def get_detail(self, detail_page_url: str):
detail_api = self._base_url + detail_page_url
resp = self.get(detail_api)
if resp.status_code != 200:
return AnimeDetailInfo()
body = self.xpath(resp.text, '//div[@id="container"]')[0] # 详细信息所在的区域
anime_detail = AnimeDetailInfo()
anime_detail.title = body.xpath(".//h4/text()")[0]
anime_detail.cover_url = "https:" + body.xpath(
'.//img[@class="poster"]/@src')[0]
anime_detail.desc = "".join(
body.xpath(
'.//div[@class="detail_imform_desc_pre"]//text()')).replace(
"\r\n", "").strip()
anime_detail.category = body.xpath(
'.//li[@class="detail_imform_kv"][9]/span[2]/text()')[0]
play_list_blocks = body.xpath(
'.//div[@class="movurl"]') # 播放列表所在的区域, 可能有多个播放列表
for i, block in enumerate(play_list_blocks, 1):
vc = VideoCollection()
vc.name = "播放列表 " + str(i)
for video_block in block.xpath('.//li'):
video = Video()
video.name = video_block.xpath("a/@title")[0]
video.raw_url = video_block.xpath("a/@href")[
0] # /play/20170172?playid=1_1
video.handler = "AgeFansVideoHandler" # 绑定视频处理器
vc.append(video)
if vc.num != 0: # 可能有空的播放列表
anime_detail.append(vc)
return anime_detail
def get_detail(self, detail_page_url: str):
url = self._base_url + detail_page_url
logger.info(f"Parsing detail page: {url}")
resp = self.get(url)
if resp.status_code != 200:
logger.warning(f"Response error: {resp.status_code} {url}")
return AnimeDetailInfo()
body = self.xpath(resp.text, '//div[@class="fire l"]')[0]
anime_detail = AnimeDetailInfo()
anime_detail.title = body.xpath("./div/h1/text()")[0]
anime_detail.category = " ".join(
body.xpath('.//div[@class="sinfo"]/span[3]/a/text()'))
anime_detail.desc = body.xpath(
'.//div[@class="info"]/text()')[0].replace("\r\n", "").strip()
anime_detail.cover_url = body.xpath(
'.//div[@class="thumb l"]/img/@src')[0]
vc = VideoCollection()
vc.name = "播放列表"
video_blocks = body.xpath('.//div[@class="movurl"]//li')
for block in video_blocks:
video = Video()
video.name = block.xpath("./a/text()")[0]
video.raw_url = block.xpath("./a/@href")[0] # '/v/3849-162.html'
video.handler = "YHDMVideoHandler"
vc.append(video)
anime_detail.append(vc)
return anime_detail
def populate_database(self, published_before_datetime, page_token, search_params):
if page_token != None: # get next page results
search_params['pageToken'] = page_token
elif published_before_datetime != None: # Continue populating database from where we last left off
search_params['publishedBefore'] = published_before_datetime.strftime('%Y-%m-%dT%H:%M:%SZ')
else: # the database is currently empty. Start a search
# Ensure that we get the updated videos list
search_params['publishedAfter'] = (datetime.now()+timedelta(days=1)).strftime('%Y-%m-%dT%H:%M:%SZ')
response = requests.get(self.request_url, params=search_params)
if str(response.status_code) == '200':
response = response.json() # Convert json response to python dictionary
next_page_token = response['nextPageToken']
for item in response['items']:
video_data = item['snippet']
video = Video() # Video Model instance
video.title = video_data['title']
video.description = video_data['description']
video.publish_datetime = video_data['publishedAt']
video.thumbnail_url = video_data['thumbnails']['default']['url']
video.save()
print(video.publish_datetime)
return next_page_token # needed for next API call
else:
print('failed to fetch data')
print(response.json())
def like_by_url(request):
if not request.user.is_authenticated():
raise Unauthorized()
querydict = request.GET if request.method == 'GET' else request.POST
try:
normalized_url = url_fix(querydict['url'])
except KeyError:
raise BadRequest('Parameter:url missing')
except MalformedURLException:
raise BadRequest('Malformed URL:%s' % url)
try:
video = Video.objects.get(url=normalized_url)
try:
UserVideo.objects.get(user=request.user, video=video, liked_timestamp__isnull=False)
except UserVideo.DoesNotExist:
push_like_to_fb.delay(video, request.user)
user_video = request.user.like_video(video)
task_id = video.task_id
except Video.DoesNotExist:
video = Video(url=normalized_url)
video.save()
user_video = UserVideo(user=request.user,
video=video,
host=request.META.get('HTTP_REFERER'),
liked=True,
liked_timestamp=datetime.utcnow())
user_video.save()
# Fetch video metadata in background
task = fetch.delay(request.user.id,
normalized_url,
user_video.host,
callback=push_like_to_fb.subtask((request.user, )))
task_id = task.task_id
info = as_dict(user_video)
info['task_id'] = task_id
info['first'] = request.user.notifications()['firstlike']
return info
def add(request):
if not request.user.is_authenticated():
raise Unauthorized()
querydict = request.GET if request.method == 'GET' else request.POST
try:
normalized_url = url_fix(querydict['url'])
except KeyError:
raise BadRequest('Parameter:url missing')
except MalformedURLException:
raise BadRequest('Malformed URL:%s' % url)
try:
user_video = UserVideo.objects.get(user=request.user, video__url=normalized_url)
if user_video.saved:
raise BadRequest('Video:%s already saved with id:%s' % (user_video.video.url, user_video.video.id))
except UserVideo.DoesNotExist:
try:
video = Video.objects.get(url=normalized_url)
except Video.DoesNotExist:
video = Video(url=normalized_url)
video.save()
user_video = UserVideo(user=request.user, video=video)
user_video.saved = True
user_video.saved_timestamp = datetime.utcnow()
user_video.host = request.META.get('HTTP_REFERER')
user_video.save()
# Fetch video metadata in background
task = fetch.delay(request.user.id, normalized_url, user_video.host)
info = as_dict(user_video)
info.update({'first': request.user.saved_videos().count() == 1,
'unwatched': request.user.unwatched_videos().count(),
'task_id': task.task_id})
return info
def get_detail(self, detail_page_url: str):
resp = self.get(self._detail_api, params={"userid": "", "videoId": detail_page_url})
if resp.status_code != 200:
logger.warning(f"Response error: {resp.status_code} {self._search_api}")
return AnimeDetailInfo()
detail = resp.json().get("data") # 视频详情信息
anime_detail = AnimeDetailInfo()
anime_detail.title = detail["videoName"]
anime_detail.cover_url = detail["videoImg"]
anime_detail.desc = detail["videoDoc"].replace("\r\n", "") # 完整的简介
anime_detail.category = detail["videoClass"]
for play_list in detail["videoSets"]:
vc = VideoCollection() # 番剧的视频列表
vc.name = play_list["load"] # 列表名, 线路 I, 线路 II
for video in play_list["list"]:
vc.append(Video(video["ji"], video["playid"], "ZZFunVideoHandler"))
anime_detail.append(vc)
return anime_detail
def get_detail(self, detail_page_url: str):
resp = self.get(self._detail_api, params={"vid": detail_page_url})
if resp.status_code != 200 or resp.json()["code"] != 1:
logger.warning(
f"Response error: {resp.status_code} {self._search_api}")
return AnimeDetailInfo()
detail = resp.json().get("data") # 视频详情信息
anime_detail = AnimeDetailInfo()
anime_detail.title = detail["name"]
anime_detail.cover_url = detail["pic"]
anime_detail.desc = detail["label"]
anime_detail.category = detail["type"]
vc = VideoCollection()
vc.name = "视频列表"
video_set = dict(detail["playUrl"])
for name, url in video_set.items():
vc.append(Video(name, url))
anime_detail.append(vc)
return anime_detail
def populate():
client = get_spotify()
errors = []
tracks = Track.objects.filter(populated=False, spotify_id__isnull=False)[:10]
for track in tracks:
try:
data = client._get("audio-features/" + client._get_id("track", track.spotify_id))
except Exception as e:
errors.append(str(e))
continue
track.analysis_url = data["analysis_url"]
track.key = data["key"]
track.time_signature = data["time_signature"]
track.danceability = data["danceability"]
track.energy = data["energy"]
track.loudness = data["loudness"]
track.speechiness = data["speechiness"]
track.acousticness = data["acousticness"]
track.instrumentalness = data["instrumentalness"]
track.liveness = data["liveness"]
track.valence = data["valence"]
track.tempo = data["tempo"]
track.duration_ms = data["duration_ms"]
track.populated = True
track.save()
youtube_videos = YouTubeHelper().search("%s - %s" % (track.artists_to_str, track.title))
for youtube_video in youtube_videos:
try:
# do not load the same video twice
Video.objects.get(video_id=youtube_video["id"]["videoId"])
continue
except Video.DoesNotExist:
pass
except Video.MultipleObjectsReturned:
continue
try:
video = Video()
video.track = track
video.source = "youtube"
video.description = youtube_video["snippet"]["description"]
video.title = youtube_video["snippet"]["title"]
video.channel_id = youtube_video["snippet"]["channelId"]
video.url = "https://www.youtube.com/watch?v=%s" % youtube_video["id"]["videoId"]
video.video_id = youtube_video["id"]["videoId"]
video.save()
except Exception as e:
errors.append(str(e))
continue
return [tracks, errors]
def report(report_id):
if request.method == 'GET':
if report_id is None:
reports = Report.query.all()
reports = [r.to_dict() for r in reports]
return jsonify({
'data': reports
})
else:
reports = Report.query.filter_by(id=report_id).first()
return jsonify({
'data': reports.to_dict()
})
elif request.method == 'POST':
data = request.json
print(data)
valid, missing = Report.validate_json(data)
if not valid:
return jsonify({
'message': '{} not given in request.'.format(', '.join(missing))
}), 422
report = Report(data['user_id'],
data['report_type'],
data['timestamp'],
data['location']['longitude'],
data['location']['latitude'])
db.session.add(report)
db.session.commit()
return jsonify(report.id)
elif request.method == 'PUT':
print(request.headers)
if 'video' not in request.files:
return jsonify({'message': 'Request does not have a file'}), 422
file = request.files['video']
if file.filename == '':
return jsonify({'message': 'Request does not have a file'}), 422
if file and utils.allowed_file(file.filename):
ext = utils.get_ext(file.filename)
file_id = uuid.uuid4().int
filename = str(file_id) + '.' + ext
abs_fpath = os.path.join(app.config['UPLOAD_FOLDER'], filename)
file.save(abs_fpath)
rel_path = os.path.join(os.path.basename(app.config['UPLOAD_FOLDER']), filename)
video = Video(path=rel_path, ext=ext)
db.session.add(video)
db.session.commit()
url = os.path.join('video', str(video.id))
video.url = url
db.session.commit()
report = Report.query.filter_by(id=report_id).first()
report.video_id = video.id
db.session.commit()
if report.report_type == 'hit':
pipelines.hit(report, abs_fpath)
elif report.report_type == 'witness':
pipelines.witness(report, abs_fpath)
return jsonify({'message': 'Successfully uploaded video.'})
elif request.method == 'DELETE':
report = Report.query.filter_by(id=report_id).first()
db.session.delete(report)
db.session.commit()
return jsonify({
'message': 'Successfully deleted.'
})
def edit_profile(data: dict = {}, profile: object = None):
"""PUT Request Parsing
Args:
data (dict, optional): PUT Request data. Defaults to {}.
profile (MongoDB Model, optional): Edits the model in place. Defaults to None.
Returns:
Boolean: True if successful otherwise false
"""
if not data or not profile:
return False
if isinstance(profile, MentorProfile):
# Edit fields or keep original data if no added data
profile.professional_title = data.get("professional_title",
profile.professional_title)
profile.specializations = data.get("specializations",
profile.specializations)
profile.offers_group_appointments = data.get(
"offers_group_appointments", profile.offers_group_appointments)
profile.offers_in_person = data.get("offers_in_person",
profile.offers_in_person)
profile.linkedin = data.get("linkedin", profile.linkedin)
profile.website = data.get("website", profile.website)
# Create video objects for each item in list
if "videos" in data:
video_data = data.get("videos")
profile.videos = [
Video(
title=video.get("title"),
url=video.get("url"),
tag=video.get("tag"),
date_uploaded=video.get("date_uploaded"),
) for video in video_data
]
elif isinstance(profile, MenteeProfile):
profile.age = data.get("age", profile.age)
profile.gender = data.get("gender", profile.gender)
profile.organization = data.get("organization", profile.organization)
profile.is_private = data.get("is_private", profile.is_private)
if "video" in data:
video_data = data.get("video")
profile.video = Video(
title=video_data.get("title"),
url=video_data.get("url"),
tag=video_data.get("tag"),
date_uploaded=video_data.get("date_uploaded"),
)
profile.name = data.get("name", profile.name)
profile.location = data.get("location", profile.location)
profile.email = data.get("email", profile.email)
profile.phone_number = data.get("phone_number", profile.phone_number)
profile.languages = data.get("languages", profile.languages)
profile.biography = data.get("biography", profile.biography)
profile.taking_appointments = data.get("taking_appointments",
profile.taking_appointments)
profile.text_notifications = data.get("text_notifications",
profile.text_notifications)
profile.email_notifications = data.get("email_notifications",
profile.email_notifications)
# Create education object
if "education" in data:
education_data = data.get("education")
profile.education = [
Education(
education_level=education.get("education_level"),
majors=education.get("majors"),
school=education.get("school"),
graduation_year=education.get("graduation_year"),
) for education in education_data
]
return True
def new_profile(data: dict = {}, profile_type: int = -1):
"""Parses data given by POST request
Args:
data (dict): POST Data. Defaults to {}.
profile_type (int): Type of account parsing. Defaults to -1
Returns:
MongoDB Model: Depending on type it returns the respective model object
"""
if not data or profile_type == -1:
return None
new_profile = None
if profile_type == Account.MENTOR:
new_profile = MentorProfile(
firebase_uid=data["firebase_uid"],
name=data["name"],
email=data["email"],
professional_title=data["professional_title"],
specializations=data["specializations"],
offers_in_person=data["offers_in_person"],
offers_group_appointments=data["offers_group_appointments"],
email_notifications=data.get("email_notifications", True),
text_notifications=data.get("text_notifications", False),
taking_appointments=data.get("taking_appointments", False),
)
new_profile.website = data.get("website")
new_profile.linkedin = data.get("linkedin")
if "videos" in data:
video_data = data.get("videos")
new_profile.videos = [
Video(
title=video["title"],
url=video["url"],
tag=video["tag"],
date_uploaded=video["date_uploaded"],
) for video in video_data
]
elif profile_type == Account.MENTEE:
new_profile = MenteeProfile(
firebase_uid=data["firebase_uid"],
name=data["name"],
# TODO: Change this to the actual email and remove default
email=data.get("email", "*****@*****.**"),
email_notifications=data.get("email_notifications", True),
text_notifications=data.get("text_notifications", False),
organization=data["organization"],
age=data["age"],
gender=data["gender"],
is_private=data.get("is_private", True),
)
if "video" in data:
video_data = data.get("video")
new_profile.video = Video(
title=video_data["title"],
url=video_data["url"],
tag=video_data["tag"],
date_uploaded=video_data["date_uploaded"],
)
else:
# There is not match with mentee/mentor
return None
new_profile.languages = data["languages"]
new_profile.biography = data.get("biography")
new_profile.phone_number = data.get("phone_number")
new_profile.location = data.get("location")
if "education" in data:
education_data = data.get("education")
new_profile.education = [
Education(
education_level=education.get("education_level"),
majors=education.get("majors"),
school=education.get("school"),
graduation_year=education.get("graduation_year"),
) for education in education_data
]
return new_profile
def populate(self):
errors = []
limit = 10
last_id = 0
total = 0
while True:
tracks = Track.objects.filter(
~Exists(Video.objects.filter(track=OuterRef('pk'))),
id__gt=last_id).order_by('id')[:limit]
for track in tracks:
youtube_videos = self.search(
"%s - %s" % (track.artists_to_str, track.title))
if len(youtube_videos) > 0:
for youtube_video in youtube_videos:
try:
# do not load the same video twice
Video.objects.get(
video_id=youtube_video["id"]["videoId"])
continue
except Video.DoesNotExist:
pass
except Video.MultipleObjectsReturned:
continue
try:
video = Video()
video.track = track
video.source = "youtube"
video.description = youtube_video["snippet"][
"description"]
video.title = youtube_video["snippet"]["title"]
video.channel_id = youtube_video["snippet"][
"channelId"]
video.url = "https://www.youtube.com/watch?v=%s" % youtube_video[
"id"]["videoId"]
video.video_id = youtube_video["id"]["videoId"]
video.save()
except Exception as e:
errors.append(str(e))
continue
else:
errors.append("No videos for track %s by %s" %
(track.title, track.artists_to_str))
last_id = track.id
total += 1
if 0 == len(tracks):
break
return [total, errors]
def post(self):
for video in Video.where(encoded_at=None):
log.info("Enqueueing for encoding", video_id=video.id)
ingest_video.delay(video.id)
return "ok", 201
