def get_detail(self, detail_page_url: str): detail_api = self._base_url + detail_page_url resp = self.get(detail_api) if resp.status_code != 200: return AnimeDetailInfo() body = self.xpath(resp.text, '//div[@id="container"]')[0] # 详细信息所在的区域 anime_detail = AnimeDetailInfo() anime_detail.title = body.xpath(".//h4/text()")[0] anime_detail.cover_url = "https:" + body.xpath( './/img[@class="poster"]/@src')[0] anime_detail.desc = "".join( body.xpath( './/div[@class="detail_imform_desc_pre"]//text()')).replace( "\r\n", "").strip() anime_detail.category = body.xpath( './/li[@class="detail_imform_kv"][9]/span[2]/text()')[0] play_list_blocks = body.xpath( './/div[@class="movurl"]') # 播放列表所在的区域, 可能有多个播放列表 for i, block in enumerate(play_list_blocks, 1): vc = VideoCollection() vc.name = "播放列表 " + str(i) for video_block in block.xpath('.//li'): video = Video() video.name = video_block.xpath("a/@title")[0] video.raw_url = video_block.xpath("a/@href")[ 0] # /play/20170172?playid=1_1 video.handler = "AgeFansVideoHandler" # 绑定视频处理器 vc.append(video) if vc.num != 0: # 可能有空的播放列表 anime_detail.append(vc) return anime_detail
def get_detail(self, detail_page_url: str): url = self._base_url + detail_page_url logger.info(f"Parsing detail page: {url}") resp = self.get(url) if resp.status_code != 200: logger.warning(f"Response error: {resp.status_code} {url}") return AnimeDetailInfo() body = self.xpath(resp.text, '//div[@class="fire l"]')[0] anime_detail = AnimeDetailInfo() anime_detail.title = body.xpath("./div/h1/text()")[0] anime_detail.category = " ".join( body.xpath('.//div[@class="sinfo"]/span[3]/a/text()')) anime_detail.desc = body.xpath( './/div[@class="info"]/text()')[0].replace("\r\n", "").strip() anime_detail.cover_url = body.xpath( './/div[@class="thumb l"]/img/@src')[0] vc = VideoCollection() vc.name = "播放列表" video_blocks = body.xpath('.//div[@class="movurl"]//li') for block in video_blocks: video = Video() video.name = block.xpath("./a/text()")[0] video.raw_url = block.xpath("./a/@href")[0] # '/v/3849-162.html' video.handler = "YHDMVideoHandler" vc.append(video) anime_detail.append(vc) return anime_detail
def populate_database(self, published_before_datetime, page_token, search_params): if page_token != None: # get next page results search_params['pageToken'] = page_token elif published_before_datetime != None: # Continue populating database from where we last left off search_params['publishedBefore'] = published_before_datetime.strftime('%Y-%m-%dT%H:%M:%SZ') else: # the database is currently empty. Start a search # Ensure that we get the updated videos list search_params['publishedAfter'] = (datetime.now()+timedelta(days=1)).strftime('%Y-%m-%dT%H:%M:%SZ') response = requests.get(self.request_url, params=search_params) if str(response.status_code) == '200': response = response.json() # Convert json response to python dictionary next_page_token = response['nextPageToken'] for item in response['items']: video_data = item['snippet'] video = Video() # Video Model instance video.title = video_data['title'] video.description = video_data['description'] video.publish_datetime = video_data['publishedAt'] video.thumbnail_url = video_data['thumbnails']['default']['url'] video.save() print(video.publish_datetime) return next_page_token # needed for next API call else: print('failed to fetch data') print(response.json())
def like_by_url(request): if not request.user.is_authenticated(): raise Unauthorized() querydict = request.GET if request.method == 'GET' else request.POST try: normalized_url = url_fix(querydict['url']) except KeyError: raise BadRequest('Parameter:url missing') except MalformedURLException: raise BadRequest('Malformed URL:%s' % url) try: video = Video.objects.get(url=normalized_url) try: UserVideo.objects.get(user=request.user, video=video, liked_timestamp__isnull=False) except UserVideo.DoesNotExist: push_like_to_fb.delay(video, request.user) user_video = request.user.like_video(video) task_id = video.task_id except Video.DoesNotExist: video = Video(url=normalized_url) video.save() user_video = UserVideo(user=request.user, video=video, host=request.META.get('HTTP_REFERER'), liked=True, liked_timestamp=datetime.utcnow()) user_video.save() # Fetch video metadata in background task = fetch.delay(request.user.id, normalized_url, user_video.host, callback=push_like_to_fb.subtask((request.user, ))) task_id = task.task_id info = as_dict(user_video) info['task_id'] = task_id info['first'] = request.user.notifications()['firstlike'] return info
def add(request): if not request.user.is_authenticated(): raise Unauthorized() querydict = request.GET if request.method == 'GET' else request.POST try: normalized_url = url_fix(querydict['url']) except KeyError: raise BadRequest('Parameter:url missing') except MalformedURLException: raise BadRequest('Malformed URL:%s' % url) try: user_video = UserVideo.objects.get(user=request.user, video__url=normalized_url) if user_video.saved: raise BadRequest('Video:%s already saved with id:%s' % (user_video.video.url, user_video.video.id)) except UserVideo.DoesNotExist: try: video = Video.objects.get(url=normalized_url) except Video.DoesNotExist: video = Video(url=normalized_url) video.save() user_video = UserVideo(user=request.user, video=video) user_video.saved = True user_video.saved_timestamp = datetime.utcnow() user_video.host = request.META.get('HTTP_REFERER') user_video.save() # Fetch video metadata in background task = fetch.delay(request.user.id, normalized_url, user_video.host) info = as_dict(user_video) info.update({'first': request.user.saved_videos().count() == 1, 'unwatched': request.user.unwatched_videos().count(), 'task_id': task.task_id}) return info
def get_detail(self, detail_page_url: str): resp = self.get(self._detail_api, params={"userid": "", "videoId": detail_page_url}) if resp.status_code != 200: logger.warning(f"Response error: {resp.status_code} {self._search_api}") return AnimeDetailInfo() detail = resp.json().get("data") # 视频详情信息 anime_detail = AnimeDetailInfo() anime_detail.title = detail["videoName"] anime_detail.cover_url = detail["videoImg"] anime_detail.desc = detail["videoDoc"].replace("\r\n", "") # 完整的简介 anime_detail.category = detail["videoClass"] for play_list in detail["videoSets"]: vc = VideoCollection() # 番剧的视频列表 vc.name = play_list["load"] # 列表名, 线路 I, 线路 II for video in play_list["list"]: vc.append(Video(video["ji"], video["playid"], "ZZFunVideoHandler")) anime_detail.append(vc) return anime_detail
def get_detail(self, detail_page_url: str): resp = self.get(self._detail_api, params={"vid": detail_page_url}) if resp.status_code != 200 or resp.json()["code"] != 1: logger.warning( f"Response error: {resp.status_code} {self._search_api}") return AnimeDetailInfo() detail = resp.json().get("data") # 视频详情信息 anime_detail = AnimeDetailInfo() anime_detail.title = detail["name"] anime_detail.cover_url = detail["pic"] anime_detail.desc = detail["label"] anime_detail.category = detail["type"] vc = VideoCollection() vc.name = "视频列表" video_set = dict(detail["playUrl"]) for name, url in video_set.items(): vc.append(Video(name, url)) anime_detail.append(vc) return anime_detail
def populate(): client = get_spotify() errors = [] tracks = Track.objects.filter(populated=False, spotify_id__isnull=False)[:10] for track in tracks: try: data = client._get("audio-features/" + client._get_id("track", track.spotify_id)) except Exception as e: errors.append(str(e)) continue track.analysis_url = data["analysis_url"] track.key = data["key"] track.time_signature = data["time_signature"] track.danceability = data["danceability"] track.energy = data["energy"] track.loudness = data["loudness"] track.speechiness = data["speechiness"] track.acousticness = data["acousticness"] track.instrumentalness = data["instrumentalness"] track.liveness = data["liveness"] track.valence = data["valence"] track.tempo = data["tempo"] track.duration_ms = data["duration_ms"] track.populated = True track.save() youtube_videos = YouTubeHelper().search("%s - %s" % (track.artists_to_str, track.title)) for youtube_video in youtube_videos: try: # do not load the same video twice Video.objects.get(video_id=youtube_video["id"]["videoId"]) continue except Video.DoesNotExist: pass except Video.MultipleObjectsReturned: continue try: video = Video() video.track = track video.source = "youtube" video.description = youtube_video["snippet"]["description"] video.title = youtube_video["snippet"]["title"] video.channel_id = youtube_video["snippet"]["channelId"] video.url = "https://www.youtube.com/watch?v=%s" % youtube_video["id"]["videoId"] video.video_id = youtube_video["id"]["videoId"] video.save() except Exception as e: errors.append(str(e)) continue return [tracks, errors]
def report(report_id): if request.method == 'GET': if report_id is None: reports = Report.query.all() reports = [r.to_dict() for r in reports] return jsonify({ 'data': reports }) else: reports = Report.query.filter_by(id=report_id).first() return jsonify({ 'data': reports.to_dict() }) elif request.method == 'POST': data = request.json print(data) valid, missing = Report.validate_json(data) if not valid: return jsonify({ 'message': '{} not given in request.'.format(', '.join(missing)) }), 422 report = Report(data['user_id'], data['report_type'], data['timestamp'], data['location']['longitude'], data['location']['latitude']) db.session.add(report) db.session.commit() return jsonify(report.id) elif request.method == 'PUT': print(request.headers) if 'video' not in request.files: return jsonify({'message': 'Request does not have a file'}), 422 file = request.files['video'] if file.filename == '': return jsonify({'message': 'Request does not have a file'}), 422 if file and utils.allowed_file(file.filename): ext = utils.get_ext(file.filename) file_id = uuid.uuid4().int filename = str(file_id) + '.' + ext abs_fpath = os.path.join(app.config['UPLOAD_FOLDER'], filename) file.save(abs_fpath) rel_path = os.path.join(os.path.basename(app.config['UPLOAD_FOLDER']), filename) video = Video(path=rel_path, ext=ext) db.session.add(video) db.session.commit() url = os.path.join('video', str(video.id)) video.url = url db.session.commit() report = Report.query.filter_by(id=report_id).first() report.video_id = video.id db.session.commit() if report.report_type == 'hit': pipelines.hit(report, abs_fpath) elif report.report_type == 'witness': pipelines.witness(report, abs_fpath) return jsonify({'message': 'Successfully uploaded video.'}) elif request.method == 'DELETE': report = Report.query.filter_by(id=report_id).first() db.session.delete(report) db.session.commit() return jsonify({ 'message': 'Successfully deleted.' })
def edit_profile(data: dict = {}, profile: object = None): """PUT Request Parsing Args: data (dict, optional): PUT Request data. Defaults to {}. profile (MongoDB Model, optional): Edits the model in place. Defaults to None. Returns: Boolean: True if successful otherwise false """ if not data or not profile: return False if isinstance(profile, MentorProfile): # Edit fields or keep original data if no added data profile.professional_title = data.get("professional_title", profile.professional_title) profile.specializations = data.get("specializations", profile.specializations) profile.offers_group_appointments = data.get( "offers_group_appointments", profile.offers_group_appointments) profile.offers_in_person = data.get("offers_in_person", profile.offers_in_person) profile.linkedin = data.get("linkedin", profile.linkedin) profile.website = data.get("website", profile.website) # Create video objects for each item in list if "videos" in data: video_data = data.get("videos") profile.videos = [ Video( title=video.get("title"), url=video.get("url"), tag=video.get("tag"), date_uploaded=video.get("date_uploaded"), ) for video in video_data ] elif isinstance(profile, MenteeProfile): profile.age = data.get("age", profile.age) profile.gender = data.get("gender", profile.gender) profile.organization = data.get("organization", profile.organization) profile.is_private = data.get("is_private", profile.is_private) if "video" in data: video_data = data.get("video") profile.video = Video( title=video_data.get("title"), url=video_data.get("url"), tag=video_data.get("tag"), date_uploaded=video_data.get("date_uploaded"), ) profile.name = data.get("name", profile.name) profile.location = data.get("location", profile.location) profile.email = data.get("email", profile.email) profile.phone_number = data.get("phone_number", profile.phone_number) profile.languages = data.get("languages", profile.languages) profile.biography = data.get("biography", profile.biography) profile.taking_appointments = data.get("taking_appointments", profile.taking_appointments) profile.text_notifications = data.get("text_notifications", profile.text_notifications) profile.email_notifications = data.get("email_notifications", profile.email_notifications) # Create education object if "education" in data: education_data = data.get("education") profile.education = [ Education( education_level=education.get("education_level"), majors=education.get("majors"), school=education.get("school"), graduation_year=education.get("graduation_year"), ) for education in education_data ] return True
def new_profile(data: dict = {}, profile_type: int = -1): """Parses data given by POST request Args: data (dict): POST Data. Defaults to {}. profile_type (int): Type of account parsing. Defaults to -1 Returns: MongoDB Model: Depending on type it returns the respective model object """ if not data or profile_type == -1: return None new_profile = None if profile_type == Account.MENTOR: new_profile = MentorProfile( firebase_uid=data["firebase_uid"], name=data["name"], email=data["email"], professional_title=data["professional_title"], specializations=data["specializations"], offers_in_person=data["offers_in_person"], offers_group_appointments=data["offers_group_appointments"], email_notifications=data.get("email_notifications", True), text_notifications=data.get("text_notifications", False), taking_appointments=data.get("taking_appointments", False), ) new_profile.website = data.get("website") new_profile.linkedin = data.get("linkedin") if "videos" in data: video_data = data.get("videos") new_profile.videos = [ Video( title=video["title"], url=video["url"], tag=video["tag"], date_uploaded=video["date_uploaded"], ) for video in video_data ] elif profile_type == Account.MENTEE: new_profile = MenteeProfile( firebase_uid=data["firebase_uid"], name=data["name"], # TODO: Change this to the actual email and remove default email=data.get("email", "*****@*****.**"), email_notifications=data.get("email_notifications", True), text_notifications=data.get("text_notifications", False), organization=data["organization"], age=data["age"], gender=data["gender"], is_private=data.get("is_private", True), ) if "video" in data: video_data = data.get("video") new_profile.video = Video( title=video_data["title"], url=video_data["url"], tag=video_data["tag"], date_uploaded=video_data["date_uploaded"], ) else: # There is not match with mentee/mentor return None new_profile.languages = data["languages"] new_profile.biography = data.get("biography") new_profile.phone_number = data.get("phone_number") new_profile.location = data.get("location") if "education" in data: education_data = data.get("education") new_profile.education = [ Education( education_level=education.get("education_level"), majors=education.get("majors"), school=education.get("school"), graduation_year=education.get("graduation_year"), ) for education in education_data ] return new_profile
def populate(self): errors = [] limit = 10 last_id = 0 total = 0 while True: tracks = Track.objects.filter( ~Exists(Video.objects.filter(track=OuterRef('pk'))), id__gt=last_id).order_by('id')[:limit] for track in tracks: youtube_videos = self.search( "%s - %s" % (track.artists_to_str, track.title)) if len(youtube_videos) > 0: for youtube_video in youtube_videos: try: # do not load the same video twice Video.objects.get( video_id=youtube_video["id"]["videoId"]) continue except Video.DoesNotExist: pass except Video.MultipleObjectsReturned: continue try: video = Video() video.track = track video.source = "youtube" video.description = youtube_video["snippet"][ "description"] video.title = youtube_video["snippet"]["title"] video.channel_id = youtube_video["snippet"][ "channelId"] video.url = "https://www.youtube.com/watch?v=%s" % youtube_video[ "id"]["videoId"] video.video_id = youtube_video["id"]["videoId"] video.save() except Exception as e: errors.append(str(e)) continue else: errors.append("No videos for track %s by %s" % (track.title, track.artists_to_str)) last_id = track.id total += 1 if 0 == len(tracks): break return [total, errors]
def post(self): for video in Video.where(encoded_at=None): log.info("Enqueueing for encoding", video_id=video.id) ingest_video.delay(video.id) return "ok", 201