def populate_database(self, published_before_datetime, page_token, search_params): if page_token != None: # get next page results search_params['pageToken'] = page_token elif published_before_datetime != None: # Continue populating database from where we last left off search_params['publishedBefore'] = published_before_datetime.strftime('%Y-%m-%dT%H:%M:%SZ') else: # the database is currently empty. Start a search # Ensure that we get the updated videos list search_params['publishedAfter'] = (datetime.now()+timedelta(days=1)).strftime('%Y-%m-%dT%H:%M:%SZ') response = requests.get(self.request_url, params=search_params) if str(response.status_code) == '200': response = response.json() # Convert json response to python dictionary next_page_token = response['nextPageToken'] for item in response['items']: video_data = item['snippet'] video = Video() # Video Model instance video.title = video_data['title'] video.description = video_data['description'] video.publish_datetime = video_data['publishedAt'] video.thumbnail_url = video_data['thumbnails']['default']['url'] video.save() print(video.publish_datetime) return next_page_token # needed for next API call else: print('failed to fetch data') print(response.json())
def populate(): client = get_spotify() errors = [] tracks = Track.objects.filter(populated=False, spotify_id__isnull=False)[:10] for track in tracks: try: data = client._get("audio-features/" + client._get_id("track", track.spotify_id)) except Exception as e: errors.append(str(e)) continue track.analysis_url = data["analysis_url"] track.key = data["key"] track.time_signature = data["time_signature"] track.danceability = data["danceability"] track.energy = data["energy"] track.loudness = data["loudness"] track.speechiness = data["speechiness"] track.acousticness = data["acousticness"] track.instrumentalness = data["instrumentalness"] track.liveness = data["liveness"] track.valence = data["valence"] track.tempo = data["tempo"] track.duration_ms = data["duration_ms"] track.populated = True track.save() youtube_videos = YouTubeHelper().search("%s - %s" % (track.artists_to_str, track.title)) for youtube_video in youtube_videos: try: # do not load the same video twice Video.objects.get(video_id=youtube_video["id"]["videoId"]) continue except Video.DoesNotExist: pass except Video.MultipleObjectsReturned: continue try: video = Video() video.track = track video.source = "youtube" video.description = youtube_video["snippet"]["description"] video.title = youtube_video["snippet"]["title"] video.channel_id = youtube_video["snippet"]["channelId"] video.url = "https://www.youtube.com/watch?v=%s" % youtube_video["id"]["videoId"] video.video_id = youtube_video["id"]["videoId"] video.save() except Exception as e: errors.append(str(e)) continue return [tracks, errors]
def populate(self): errors = [] limit = 10 last_id = 0 total = 0 while True: tracks = Track.objects.filter( ~Exists(Video.objects.filter(track=OuterRef('pk'))), id__gt=last_id).order_by('id')[:limit] for track in tracks: youtube_videos = self.search( "%s - %s" % (track.artists_to_str, track.title)) if len(youtube_videos) > 0: for youtube_video in youtube_videos: try: # do not load the same video twice Video.objects.get( video_id=youtube_video["id"]["videoId"]) continue except Video.DoesNotExist: pass except Video.MultipleObjectsReturned: continue try: video = Video() video.track = track video.source = "youtube" video.description = youtube_video["snippet"][ "description"] video.title = youtube_video["snippet"]["title"] video.channel_id = youtube_video["snippet"][ "channelId"] video.url = "https://www.youtube.com/watch?v=%s" % youtube_video[ "id"]["videoId"] video.video_id = youtube_video["id"]["videoId"] video.save() except Exception as e: errors.append(str(e)) continue else: errors.append("No videos for track %s by %s" % (track.title, track.artists_to_str)) last_id = track.id total += 1 if 0 == len(tracks): break return [total, errors]