def createEventMediaSuggestion(cls, author_account_key, media_url, event_key, private_details_json=None):
"""Create an Event Media Suggestion. Returns status (success, suggestion_exists, media_exists, bad_url)"""
media_dict = MediaParser.partial_media_dict_from_url(media_url)
if media_dict is not None:
if media_dict['media_type_enum'] != MediaType.YOUTUBE_VIDEO:
return 'bad_url', None
existing_media = Media.get_by_id(Media.render_key_name(media_dict['media_type_enum'], media_dict['foreign_key']))
if existing_media is None or event_key not in [reference.id() for reference in existing_media.references]:
foreign_type = Media.SLUG_NAMES[media_dict['media_type_enum']]
suggestion_id = Suggestion.render_media_key_name(event_key[:4], 'event', event_key, foreign_type, media_dict['foreign_key'])
suggestion = Suggestion.get_by_id(suggestion_id)
if not suggestion or suggestion.review_state != Suggestion.REVIEW_PENDING:
media_dict['year'] = event_key[:4]
media_dict['reference_type'] = 'event'
media_dict['reference_key'] = event_key
target_model = 'event_media'
if private_details_json is not None:
media_dict['private_details_json'] = private_details_json
suggestion = Suggestion(
id=suggestion_id,
author=author_account_key,
target_model=target_model,
)
suggestion.contents = media_dict
suggestion.put()
return 'success', suggestion
else:
return 'suggestion_exists', None
else:
return 'media_exists', None
else:
return 'bad_url', None
def create_target_model(self, suggestion):
# Setup
to_replace = None
to_replace_id = self.request.POST.get("replace-preferred-{}".format(suggestion.key.id()), None)
# Remove preferred reference from another Media if specified
team_reference = Media.create_reference(
suggestion.contents["reference_type"], suggestion.contents["reference_key"]
)
if to_replace_id:
to_replace = Media.get_by_id(to_replace_id)
if team_reference not in to_replace.preferred_references:
return # Preferred reference must have been edited earlier. Skip this Suggestion for now.
to_replace.preferred_references.remove(team_reference)
# Add preferred reference to current Media (images only) if explicitly listed in preferred_keys or if to_replace_id exists
media_type_enum = suggestion.contents["media_type_enum"]
preferred_references = []
if media_type_enum in MediaType.image_types and (
"preferred::{}".format(suggestion.key.id()) in self.preferred_keys or to_replace_id
):
preferred_references = [team_reference]
media = MediaCreator.create_media_model(suggestion, team_reference, preferred_references)
# Mark Suggestion as accepted
suggestion.review_state = Suggestion.REVIEW_ACCEPTED
suggestion.reviewer = self.user_bundle.account.key
suggestion.reviewed_at = datetime.datetime.now()
# Do all DB writes
if to_replace:
MediaManipulator.createOrUpdate(to_replace, auto_union=False)
MediaManipulator.createOrUpdate(media)
def parse(self, response):
"""
Parse team avatar from FMSAPI
"""
# Get team avatar
current_page = response['pageCurrent']
total_pages = response['pageTotal']
teams = response['teams']
avatars = []
keys_to_delete = set()
for teamData in teams:
team_number = teamData['teamNumber']
foreign_key = "avatar_{}_frc{}".format(self.year, team_number)
media_key = ndb.Key(
'Media', Media.render_key_name(MediaType.AVATAR, foreign_key))
if not teamData['encodedAvatar']:
keys_to_delete.add(media_key)
continue
else:
encoded_avatar = teamData['encodedAvatar']
avatar = Media(
key=media_key,
details_json=json.dumps({'base64Image': encoded_avatar}),
foreign_key=foreign_key,
media_type_enum=MediaType.AVATAR,
references=[ndb.Key('Team', "frc{}".format(team_number))],
year=self.year)
avatars.append(avatar)
return (avatars, keys_to_delete, (current_page < total_pages))
def _process_request(self, request, event_key):
try:
video_list = json.loads(request.body)
except Exception:
self._errors = json.dumps({"Error": "Invalid JSON. Please check input."})
self.abort(400)
return
if not isinstance(video_list, list) or not video_list:
self._errors = json.dumps({"Error": "Invalid JSON. Please check input."})
self.abort(400)
return
media_to_put = []
event_reference = Media.create_reference('event', self.event.key_name)
for youtube_id in video_list:
media = Media(
id=Media.render_key_name(MediaType.YOUTUBE_VIDEO, youtube_id),
foreign_key=youtube_id,
media_type_enum=MediaType.YOUTUBE_VIDEO,
details_json=None,
private_details_json=None,
year=self.event.year,
references=[event_reference],
preferred_references=[],
)
media_to_put.append(media)
MediaManipulator.createOrUpdate(media_to_put)
def get_media_and_team_ref(self, team_number):
media_key_name = self.request.get("media_key_name")
media = Media.get_by_id(media_key_name)
if not media:
self.abort(400)
team_ref = Media.create_reference('team', 'frc{}'.format(team_number))
return media, team_ref
def post(self):
self._require_admin()
media_dict = MediaParser.partial_media_dict_from_url(
self.request.get('media_url').strip())
if media_dict is not None:
year_str = self.request.get('year')
if year_str == '':
year = None
else:
year = int(year_str.strip())
media = Media(
id=Media.render_key_name(media_dict['media_type_enum'],
media_dict['foreign_key']),
foreign_key=media_dict['foreign_key'],
media_type_enum=media_dict['media_type_enum'],
details_json=media_dict.get('details_json', None),
year=year,
references=[
Media.create_reference(self.request.get('reference_type'),
self.request.get('reference_key'))
],
)
MediaManipulator.createOrUpdate(media)
self.redirect(self.request.get('originating_url'))
def create_target_model(self, suggestion):
# Setup
to_replace = None
to_replace_id = self.request.POST.get('replace-preferred-{}'.format(suggestion.key.id()), None)
year = int(self.request.POST.get('year-{}'.format(suggestion.key.id())))
# Override year if necessary
suggestion.contents['year'] = year
suggestion.contents_json = json.dumps(suggestion.contents)
suggestion._contents = None
# Remove preferred reference from another Media if specified
team_reference = Media.create_reference(
suggestion.contents['reference_type'],
suggestion.contents['reference_key'])
if to_replace_id:
to_replace = Media.get_by_id(to_replace_id)
if team_reference not in to_replace.preferred_references:
# Preferred reference must have been edited earlier. Skip this Suggestion for now.
return
to_replace.preferred_references.remove(team_reference)
# Add preferred reference to current Media (images only) if explicitly listed in preferred_keys or if to_replace_id exists
media_type_enum = suggestion.contents['media_type_enum']
preferred_references = []
if media_type_enum in MediaType.image_types and ('preferred::{}'.format(suggestion.key.id()) in self.preferred_keys or to_replace_id):
preferred_references = [team_reference]
media = MediaCreator.create_media_model(suggestion, team_reference, preferred_references)
# Do all DB writes
if to_replace:
MediaManipulator.createOrUpdate(to_replace, auto_union=False)
return MediaManipulator.createOrUpdate(media)
def get_media_and_team_ref(self, team_number):
media_key_name = self.request.get("media_key_name")
media = Media.get_by_id(media_key_name)
if not media:
self.abort(400)
team_ref = Media.create_reference('team', 'frc{}'.format(team_number))
return media, team_ref
def createEventMediaSuggestion(cls, author_account_key, media_url, event_key, private_details_json=None):
"""Create an Event Media Suggestion. Returns status (success, suggestion_exists, media_exists, bad_url)"""
media_dict = MediaParser.partial_media_dict_from_url(media_url)
if media_dict is not None:
if media_dict['media_type_enum'] != MediaType.YOUTUBE_VIDEO:
return 'bad_url', None
existing_media = Media.get_by_id(Media.render_key_name(media_dict['media_type_enum'], media_dict['foreign_key']))
if existing_media is None or event_key not in [reference.id() for reference in existing_media.references]:
foreign_type = Media.SLUG_NAMES[media_dict['media_type_enum']]
suggestion_id = Suggestion.render_media_key_name(event_key[:4], 'event', event_key, foreign_type, media_dict['foreign_key'])
suggestion = Suggestion.get_by_id(suggestion_id)
if not suggestion or suggestion.review_state != Suggestion.REVIEW_PENDING:
media_dict['year'] = event_key[:4]
media_dict['reference_type'] = 'event'
media_dict['reference_key'] = event_key
target_model = 'event_media'
if private_details_json is not None:
media_dict['private_details_json'] = private_details_json
suggestion = Suggestion(
id=suggestion_id,
author=author_account_key,
target_model=target_model,
)
suggestion.contents = media_dict
suggestion.put()
return 'success', suggestion
else:
return 'suggestion_exists', None
else:
return 'media_exists', None
else:
return 'bad_url', None
def createTeamMediaSuggestion(cls, author_account_key, media_url, team_key, year_str):
"""Create a Team Media Suggestion. Returns status (success, exists, bad_url)"""
media_dict = MediaParser.partial_media_dict_from_url(media_url)
if media_dict is not None:
existing_media = Media.get_by_id(Media.render_key_name(media_dict['media_type_enum'], media_dict['foreign_key']))
if existing_media is None or team_key not in [reference.id() for reference in existing_media.references]:
foreign_type = Media.SLUG_NAMES[media_dict['media_type_enum']]
suggestion_id = Suggestion.render_key_name(year_str, 'team', team_key, foreign_type, media_dict['foreign_key'])
suggestion = Suggestion.get_by_id(suggestion_id)
if not suggestion or suggestion.review_state != Suggestion.REVIEW_PENDING:
media_dict['year'] = int(year_str)
media_dict['reference_type'] = 'team'
media_dict['reference_key'] = team_key
suggestion = Suggestion(
id=suggestion_id,
author=author_account_key,
target_model="media",
)
suggestion.contents = media_dict
suggestion.put()
return 'success'
else:
return 'suggestion_exists'
else:
return 'media_exists'
else:
return 'bad_url'
def post(self):
team_number = self.request.get("team_number")
if not team_number:
self.abort(400)
team_number = int(team_number)
self._require_team_admin_access(team_number)
media_key_name = self.request.get("media_key_name")
media = Media.get_by_id(media_key_name)
if not media:
self.abort(400)
team_ref = Media.create_reference('team', 'frc{}'.format(team_number))
action = self.request.get('action')
if action == "remove_media_reference":
if team_ref in media.references:
media.references.remove(team_ref)
if team_ref in media.preferred_references:
media.preferred_references.remove(team_ref)
elif action == "remove_media_preferred":
if team_ref in media.preferred_references:
media.preferred_references.remove(team_ref)
elif action == "add_media_preferred":
if team_ref not in media.preferred_references:
media.preferred_references.append(team_ref)
else:
self.abort(400)
MediaManipulator.createOrUpdate(media, auto_union=False)
self.redirect('/mod/')
class TestTeamMediaApiController(unittest2.TestCase):
def setUp(self):
app = webapp2.WSGIApplication([webapp2.Route(r'/<team_key:>/<year:>', ApiTeamMediaController, methods=['GET'])], debug=True)
self.testapp = webtest.TestApp(app)
self.testbed = testbed.Testbed()
self.testbed.activate()
self.testbed.init_datastore_v3_stub()
self.testbed.init_urlfetch_stub()
self.testbed.init_memcache_stub()
self.testbed.init_taskqueue_stub(root_path=".")
self.team = Team(
id="frc254",
name="very long name",
team_number=254,
nickname="Teh Chezy Pofs",
address="Greenville, SC, USA"
)
self.team.put()
self.cdmedia = Media(
key=ndb.Key('Media', 'cdphotothread_39894'),
details_json=u'{"image_partial": "fe3/fe38d320428adf4f51ac969efb3db32c_l.jpg"}',
foreign_key=u'39894',
media_type_enum=1,
references=[ndb.Key('Team', 'frc254')],
year=2014)
self.cdmedia.put()
self.cddetails = dict()
self.cddetails["image_partial"] = "fe3/fe38d320428adf4f51ac969efb3db32c_l.jpg"
self.ytmedia = Media(
key=ndb.Key('Media', 'youtube_aFZy8iibMD0'),
details_json=None,
foreign_key=u'aFZy8iibMD0',
media_type_enum=0,
references=[ndb.Key('Team', 'frc254')],
year=2014)
self.ytmedia.put()
def tearDown(self):
self.testbed.deactivate()
def testTeamMediaApi(self):
response = self.testapp.get('/frc254/2014', headers={"X-TBA-App-Id": "tba-tests:team_media-controller-test:v01"})
media = json.loads(response.body)
self.assertEqual(len(media), 2)
cd = media[0]
self.assertEqual(cd["type"], "cdphotothread")
self.assertEqual(cd["foreign_key"], "39894")
self.assertEqual(cd["details"], self.cddetails)
yt = media[1]
self.assertEqual(yt["type"], "youtube")
self.assertEqual(yt["foreign_key"], "aFZy8iibMD0")
self.assertEqual(yt["details"], {})
def test_get_media_list_filtered_by_consumed_state(self):
user = User('testname', 'P@ssw0rd')
db.session.add(user)
db.session.commit()
media1 = Media('testmedianame1', user.id, consumed_state='finished')
media2 = Media('testmedianame2', user.id, consumed_state='not started')
media3 = Media('testmedianame3', user.id, consumed_state='started')
media4 = Media('testmedianame4', user.id, consumed_state='finished')
media5 = Media('testmedianame5', user.id, consumed_state='started')
db.session.add(media1)
db.session.add(media2)
db.session.add(media3)
db.session.add(media4)
db.session.add(media5)
db.session.commit()
not_started_media_list = sorted(get_media('testname', consumed_state='not started'),
key=lambda media: media.medianame)
started_media_list = sorted(get_media('testname', consumed_state='started'),
key=lambda media: media.medianame)
finished_media_list = sorted(get_media('testname', consumed_state='finished'),
key=lambda media: media.medianame)
self.assertListEqual(not_started_media_list, [media2])
self.assertListEqual(started_media_list, [media3, media5])
self.assertListEqual(finished_media_list, [media1, media4])
def test_get_media_multiple_users(self):
"""
'testname1': ['testmedianame1', 'testmedianame2', 'testmedianame3']
'testname2': ['testmedianame4', 'testmedianame1']
"""
user1 = User('testname1', 'P@ssw0rd')
user2 = User('testname2', 'P@ssw0rd')
db.session.add(user1)
db.session.add(user2)
db.session.commit()
media1 = Media('testmedianame1', user1.id)
media2 = Media('testmedianame2', user1.id)
media3 = Media('testmedianame3', user1.id)
media4 = Media('testmedianame4', user2.id)
media5 = Media('testmedianame1', user2.id)
db.session.add(media1)
db.session.add(media2)
db.session.add(media3)
db.session.add(media4)
db.session.add(media5)
db.session.commit()
user1_media_list = sorted(get_media('testname1'), key=lambda media: media.medianame)
self.assertListEqual(user1_media_list, [media1, media2, media3])
user2_media_list = sorted(get_media('testname2'), key=lambda media: media.medianame)
self.assertListEqual(user2_media_list, [media5, media4])
def create_target_model(self, suggestion):
# Setup
to_replace = None
to_replace_id = self.request.POST.get('replace-preferred-{}'.format(suggestion.key.id()), None)
year = int(self.request.POST.get('year-{}'.format(suggestion.key.id())))
# Override year if necessary
suggestion.contents['year'] = year
suggestion.contents_json = json.dumps(suggestion.contents)
suggestion._contents = None
# Remove preferred reference from another Media if specified
team_reference = Media.create_reference(
suggestion.contents['reference_type'],
suggestion.contents['reference_key'])
if to_replace_id:
to_replace = Media.get_by_id(to_replace_id)
if team_reference not in to_replace.preferred_references:
# Preferred reference must have been edited earlier. Skip this Suggestion for now.
return
to_replace.preferred_references.remove(team_reference)
# Add preferred reference to current Media (images only) if explicitly listed in preferred_keys or if to_replace_id exists
media_type_enum = suggestion.contents['media_type_enum']
preferred_references = []
if media_type_enum in MediaType.image_types and ('preferred::{}'.format(suggestion.key.id()) in self.preferred_keys or to_replace_id):
preferred_references = [team_reference]
media = MediaCreator.create_media_model(suggestion, team_reference, preferred_references)
# Do all DB writes
if to_replace:
MediaManipulator.createOrUpdate(to_replace, auto_union=False)
return MediaManipulator.createOrUpdate(media)
def get(self):
super(SuggestTeamMediaReviewController, self).get()
suggestions = Suggestion.query().filter(
Suggestion.review_state == Suggestion.REVIEW_PENDING).filter(
Suggestion.target_model == "media").fetch(limit=50)
# Quick and dirty way to group images together
suggestions = sorted(
suggestions,
key=lambda x: 0
if x.contents['media_type_enum'] in MediaType.image_types else 1)
reference_keys = []
existing_preferred_keys_futures = []
for suggestion in suggestions:
reference_key = suggestion.contents['reference_key']
reference = Media.create_reference(
suggestion.contents['reference_type'], reference_key)
reference_keys.append(reference)
if 'details_json' in suggestion.contents:
suggestion.details = json.loads(
suggestion.contents['details_json'])
if 'image_partial' in suggestion.details:
suggestion.details['thumbnail'] = suggestion.details[
'image_partial'].replace('_l', '_m')
# Find existing preferred images
existing_preferred_keys_futures.append(
Media.query(
Media.media_type_enum.IN(MediaType.image_types),
Media.references == reference,
Media.preferred_references == reference,
Media.year == suggestion.contents['year'],
).fetch_async(keys_only=True))
reference_futures = ndb.get_multi_async(reference_keys)
existing_preferred_futures = map(
lambda x: ndb.get_multi_async(x.get_result()),
existing_preferred_keys_futures)
references = map(lambda r: r.get_result(), reference_futures)
existing_preferred = map(lambda l: map(lambda x: x.get_result(), l),
existing_preferred_futures)
suggestions_and_references_and_preferred = zip(suggestions, references,
existing_preferred)
self.template_values.update({
"suggestions_and_references_and_preferred":
suggestions_and_references_and_preferred,
"max_preferred":
Media.MAX_PREFERRED,
})
self.response.out.write(
jinja2_engine.render(
'suggestions/suggest_team_media_review_list.html',
self.template_values))
def createTeamMediaSuggestion(cls,
author_account_key,
media_url,
team_key,
year_str,
private_details_json=None,
is_social=False):
"""Create a Team Media Suggestion. Returns status (success, suggestion_exists, media_exists, bad_url)"""
media_dict = MediaParser.partial_media_dict_from_url(media_url)
if media_dict is not None:
if media_dict.get("is_social", False) != is_social:
return 'bad_url', None
existing_media = Media.get_by_id(
Media.render_key_name(media_dict['media_type_enum'],
media_dict['foreign_key']))
if existing_media is None or team_key not in [
reference.id() for reference in existing_media.references
]:
foreign_type = Media.SLUG_NAMES[media_dict['media_type_enum']]
suggestion_id = Suggestion.render_media_key_name(
year_str, 'team', team_key, foreign_type,
media_dict['foreign_key'])
suggestion = Suggestion.get_by_id(suggestion_id)
if not suggestion or suggestion.review_state != Suggestion.REVIEW_PENDING:
media_dict['year'] = int(year_str) if year_str else None
media_dict['reference_type'] = 'team'
media_dict['reference_key'] = team_key
if private_details_json is not None:
media_dict[
'private_details_json'] = private_details_json
target_model = "media"
if media_dict.get("is_social", False):
target_model = "social-media"
if media_dict.get('media_type',
'') in MediaType.robot_types:
target_model = "robot"
if Event.validate_key_name(team_key):
target_model = 'event_media'
media_dict['reference_type'] = 'event'
suggestion = Suggestion(
id=suggestion_id,
author=author_account_key,
target_model=target_model,
)
suggestion.contents = media_dict
suggestion.put()
return 'success', suggestion
else:
return 'suggestion_exists', None
else:
return 'media_exists', None
else:
return 'bad_url', None
def testMediaExists(self):
media_id = Media.render_key_name(MediaType.IMGUR, 'ruRAxDm')
Media.get_or_insert(media_id,
media_type_enum=MediaType.IMGUR,
foreign_key='ruRAxDm',
references=[ndb.Key(Team, 'frc1124')]).put()
status, _ = SuggestionCreator.createTeamMediaSuggestion(
self.account.key, "http://imgur.com/ruRAxDm", "frc1124", "2016")
self.assertEqual(status, 'media_exists')
def testMediaExists(self):
media_id = Media.render_key_name(MediaType.YOUTUBE_VIDEO,
'H-54KMwMKY0')
Media.get_or_insert(media_id,
media_type_enum=MediaType.YOUTUBE_VIDEO,
foreign_key='H-54KMwMKY0',
references=[ndb.Key(Event, '2016nyny')]).put()
status, _ = SuggestionCreator.createEventMediaSuggestion(
self.account.key, "https://www.youtube.com/watch?v=H-54KMwMKY0",
"2016nyny")
self.assertEqual(status, 'media_exists')
def create_media(cls, suggestion, team_reference, preferred_references=[]):
media_type_enum = suggestion.contents['media_type_enum']
return Media(
id=Media.render_key_name(media_type_enum, suggestion.contents['foreign_key']),
foreign_key=suggestion.contents['foreign_key'],
media_type_enum=media_type_enum,
details_json=suggestion.contents.get('details_json', None),
private_details_json=suggestion.contents.get('private_details_json', None),
year=int(suggestion.contents['year']) if not suggestion.contents.get('is_social', False) else None,
references=[team_reference],
preferred_references=preferred_references,
)
def test_media_exists(self):
media_id = Media.render_key_name(MediaType.YOUTUBE_VIDEO, 'H-54KMwMKY0')
Media.get_or_insert(
media_id,
media_type_enum=MediaType.YOUTUBE_VIDEO,
foreign_key='H-54KMwMKY0',
references=[ndb.Key(Event, '2016nyny')]).put()
status, _ = SuggestionCreator.createEventMediaSuggestion(
self.account.key,
"https://www.youtube.com/watch?v=H-54KMwMKY0",
"2016nyny")
self.assertEqual(status, 'media_exists')
def get(self):
suggestions = (
Suggestion.query()
.filter(Suggestion.review_state == Suggestion.REVIEW_PENDING)
.filter(Suggestion.target_model == "media")
.fetch(limit=50)
)
# Quick and dirty way to group images together
suggestions = sorted(
suggestions, key=lambda x: 0 if x.contents["media_type_enum"] in MediaType.image_types else 1
)
reference_keys = []
existing_preferred_keys_futures = []
for suggestion in suggestions:
reference_key = suggestion.contents["reference_key"]
reference = Media.create_reference(suggestion.contents["reference_type"], reference_key)
reference_keys.append(reference)
if "details_json" in suggestion.contents:
suggestion.details = json.loads(suggestion.contents["details_json"])
if "image_partial" in suggestion.details:
suggestion.details["thumbnail"] = suggestion.details["image_partial"].replace("_l", "_m")
# Find existing preferred images
existing_preferred_keys_futures.append(
Media.query(
Media.media_type_enum.IN(MediaType.image_types),
Media.references == reference,
Media.preferred_references == reference,
Media.year == suggestion.contents["year"],
).fetch_async(keys_only=True)
)
reference_futures = ndb.get_multi_async(reference_keys)
existing_preferred_futures = map(lambda x: ndb.get_multi_async(x.get_result()), existing_preferred_keys_futures)
references = map(lambda r: r.get_result(), reference_futures)
existing_preferred = map(lambda l: map(lambda x: x.get_result(), l), existing_preferred_futures)
suggestions_and_references_and_preferred = zip(suggestions, references, existing_preferred)
self.template_values.update(
{
"suggestions_and_references_and_preferred": suggestions_and_references_and_preferred,
"max_preferred": Media.MAX_PREFERRED,
}
)
self.response.out.write(
jinja2_engine.render("suggestions/suggest_team_media_review_list.html", self.template_values)
)
def _process_accepted(self, accept_key, preferred_keys):
"""
Performs all actions for an accepted Suggestion in a Transaction.
Suggestions are processed one at a time (instead of in batch) in a
Transaction to prevent possible race conditions.
Actions include:
- Creating and saving a new Media for the Suggestion
- Removing a reference from another Media's preferred_references
- Marking the Suggestion as accepted and saving it
"""
# Async get
suggestion_future = Suggestion.get_by_id_async(accept_key)
# Setup
to_replace_id = self.request.POST.get('replace-preferred-{}'.format(accept_key), None)
# Resolve async Futures
suggestion = suggestion_future.get_result()
# Make sure Suggestion hasn't been processed (by another thread)
if suggestion.review_state != Suggestion.REVIEW_PENDING:
return
# Remove preferred reference from another Media if specified
team_reference = Media.create_reference(
suggestion.contents['reference_type'],
suggestion.contents['reference_key'])
if to_replace_id:
to_replace = Media.get_by_id(to_replace_id)
if team_reference not in to_replace.preferred_references:
return # Preferred reference must have been edited earlier. Skip this Suggestion for now.
to_replace.preferred_references.remove(team_reference)
# Add preferred reference to current Media (images only) if explicitly listed in preferred_keys or if to_replace_id exists
media_type_enum = suggestion.contents['media_type_enum']
preferred_references = []
if media_type_enum in MediaType.image_types and ('preferred::{}'.format(suggestion.key.id()) in preferred_keys or to_replace_id):
preferred_references = [team_reference]
media = MediaCreator.create_media(suggestion, team_reference, preferred_references)
# Mark Suggestion as accepted
suggestion.review_state = Suggestion.REVIEW_ACCEPTED
suggestion.reviewer = self.user_bundle.account.key
suggestion.reviewed_at = datetime.datetime.now()
# Do all DB writes
if to_replace_id:
MediaManipulator.createOrUpdate(to_replace, auto_union=False)
MediaManipulator.createOrUpdate(media)
suggestion.put()
def testMediaExists(self):
media_id = Media.render_key_name(MediaType.IMGUR, 'ruRAxDm')
Media.get_or_insert(
media_id,
media_type_enum=MediaType.IMGUR,
foreign_key='ruRAxDm',
references=[ndb.Key(Team, 'frc1124')]).put()
status = SuggestionCreator.createTeamMediaSuggestion(
self.account.key,
"http://imgur.com/ruRAxDm",
"frc1124",
"2016")
self.assertEqual(status, 'media_exists')
def setUp(self):
app = webapp2.WSGIApplication([
webapp2.Route(r'/<team_key:>/<year:>',
ApiTeamMediaController,
methods=['GET'])
],
debug=True)
self.testapp = webtest.TestApp(app)
self.testbed = testbed.Testbed()
self.testbed.activate()
self.testbed.init_datastore_v3_stub()
self.testbed.init_urlfetch_stub()
self.testbed.init_memcache_stub()
ndb.get_context().clear_cache(
) # Prevent data from leaking between tests
self.testbed.init_taskqueue_stub(root_path=".")
self.team = Team(
id="frc254",
name="very long name",
team_number=254,
nickname="Teh Chezy Pofs",
city="Greenville",
state_prov="SC",
country="USA",
)
self.team.put()
self.cdmedia = Media(
key=ndb.Key('Media', 'cdphotothread_39894'),
details_json=
u'{"image_partial": "fe3/fe38d320428adf4f51ac969efb3db32c_l.jpg"}',
foreign_key=u'39894',
media_type_enum=1,
references=[ndb.Key('Team', 'frc254')],
year=2014)
self.cdmedia.put()
self.cddetails = dict()
self.cddetails[
"image_partial"] = "fe3/fe38d320428adf4f51ac969efb3db32c_l.jpg"
self.ytmedia = Media(key=ndb.Key('Media', 'youtube_aFZy8iibMD0'),
details_json=None,
foreign_key=u'aFZy8iibMD0',
media_type_enum=0,
references=[ndb.Key('Team', 'frc254')],
year=2014)
self.ytmedia.put()
def test_get_media_multiple_elements(self):
user = User('testname', 'P@ssw0rd')
db.session.add(user)
db.session.commit()
media1 = Media('testmedianame1', user.id)
media2 = Media('testmedianame2', user.id)
db.session.add(media1)
db.session.add(media2)
db.session.commit()
media_list = sorted(get_media('testname'), key=lambda media: media.medianame)
self.assertListEqual(media_list, [media1, media2])
def post(self, **kwargs):
"""Base endpoint"""
file = request.files["file"]
try:
media = Media(file=file)
except InvalidMediaTypeException as e:
return {"message": str(e)}, 400
optimized_file_names = media.optimize()
for file_name in optimized_file_names:
upload(file_name)
os.remove(file_name)
media.url = CDN_URI.format(optimized_file_names[0])
media.url_optimized = CDN_URI.format(optimized_file_names[1])
if len(optimized_file_names) is 3:
media.url_poster = CDN_URI.format(optimized_file_names[2])
media.showcase = request.args.get("showcase") is not None
try:
db.session.add(media)
db.session.commit()
except Exception as e:
capture_exception(e)
return {
"message":
"An unexpected error occoured while writing to the database"
}, 400
return media.to_dict(), 201
def test_update_media_description(self):
user = User('testname', 'P@ssw0rd')
db.session.add(user)
db.session.commit()
media = Media('testmedianame', user.id, description='asdf')
db.session.add(media)
db.session.commit()
self.assertEqual(media.description, 'asdf')
media.description = 'some description'
db.session.commit()
self.assertEqual(media.description, 'some description')
def test_update_media_medium(self):
user = User('testname', 'P@ssw0rd')
db.session.add(user)
db.session.commit()
media = Media('testmedianame', user.id, medium='literature')
db.session.add(media)
db.session.commit()
self.assertEqual(media.medium, 'literature')
media.medium = 'audio'
db.session.commit()
self.assertEqual(media.medium, 'audio')
def test_update_media_order(self):
user = User('testname', 'P@ssw0rd')
db.session.add(user)
db.session.commit()
media = Media('testmedianame', user.id, order=2)
db.session.add(media)
db.session.commit()
self.assertEqual(media.order, 2)
media.order = 34
db.session.commit()
self.assertEqual(media.order, 34)
def test_update_media_consumed_state(self):
user = User('testname', 'P@ssw0rd')
db.session.add(user)
db.session.commit()
media = Media('testmedianame', user.id)
db.session.add(media)
db.session.commit()
self.assertEqual(media.consumed_state, 'not started')
media.consumed_state = 'finished'
db.session.commit()
self.assertEqual(media.consumed_state, 'finished')
def get(self):
df = DatafeedResourceLibrary()
teams = df.getHallOfFameTeams()
if teams:
media_to_update = []
for team in teams:
team_reference = Media.create_reference('team', team['team_id'])
video_foreign_key = team['video']
if video_foreign_key:
media_to_update.append(Media(id=Media.render_key_name(MediaType.YOUTUBE_VIDEO, video_foreign_key),
media_type_enum=MediaType.YOUTUBE_VIDEO,
media_tag_enum=[MediaTag.CHAIRMANS_VIDEO],
references=[team_reference],
year=team['year'],
foreign_key=video_foreign_key))
presentation_foreign_key = team['presentation']
if presentation_foreign_key:
media_to_update.append(Media(id=Media.render_key_name(MediaType.YOUTUBE_VIDEO, presentation_foreign_key),
media_type_enum=MediaType.YOUTUBE_VIDEO,
media_tag_enum=[MediaTag.CHAIRMANS_PRESENTATION],
references=[team_reference],
year=team['year'],
foreign_key=presentation_foreign_key))
essay_foreign_key = team['essay']
if essay_foreign_key:
media_to_update.append(Media(id=Media.render_key_name(MediaType.EXTERNAL_LINK, essay_foreign_key),
media_type_enum=MediaType.EXTERNAL_LINK,
media_tag_enum=[MediaTag.CHAIRMANS_ESSAY],
references=[team_reference],
year=team['year'],
foreign_key=essay_foreign_key))
MediaManipulator.createOrUpdate(media_to_update)
else:
logging.info("No Hall of Fame teams found")
teams = []
template_values = {
'teams': teams,
}
if 'X-Appengine-Taskname' not in self.request.headers: # Only write out if not in taskqueue
path = os.path.join(os.path.dirname(__file__), '../templates/datafeeds/hall_of_fame_teams_get.html')
self.response.out.write(template.render(path, template_values))
def setUp(self):
app = webapp2.WSGIApplication(
[webapp2.Route(r"/<team_key:>/<year:>", ApiTeamMediaController, methods=["GET"])], debug=True
)
self.testapp = webtest.TestApp(app)
self.testbed = testbed.Testbed()
self.testbed.activate()
self.testbed.init_datastore_v3_stub()
self.testbed.init_urlfetch_stub()
self.testbed.init_memcache_stub()
ndb.get_context().clear_cache() # Prevent data from leaking between tests
self.testbed.init_taskqueue_stub(root_path=".")
self.team = Team(
id="frc254",
name="very long name",
team_number=254,
nickname="Teh Chezy Pofs",
city="Greenville",
state_prov="SC",
country="USA",
)
self.team.put()
self.cdmedia = Media(
key=ndb.Key("Media", "cdphotothread_39894"),
details_json=u'{"image_partial": "fe3/fe38d320428adf4f51ac969efb3db32c_l.jpg"}',
foreign_key=u"39894",
media_type_enum=1,
references=[ndb.Key("Team", "frc254")],
year=2014,
)
self.cdmedia.put()
self.cddetails = dict()
self.cddetails["image_partial"] = "fe3/fe38d320428adf4f51ac969efb3db32c_l.jpg"
self.ytmedia = Media(
key=ndb.Key("Media", "youtube_aFZy8iibMD0"),
details_json=None,
foreign_key=u"aFZy8iibMD0",
media_type_enum=0,
references=[ndb.Key("Team", "frc254")],
year=2014,
)
self.ytmedia.put()
def suggest_team_media(self, request):
current_user = endpoints.get_current_user()
if current_user is None:
return BaseResponse(code=401,
message="Unauthorized to make suggestions")
user_id = PushHelper.user_email_to_id(current_user.email())
# For now, only allow team media suggestions
if request.reference_type != "team":
# Trying to suggest a media for an invalid model type
return BaseResponse(code=400, message="Bad model type")
media_dict = MediaParser.partial_media_dict_from_url(
request.media_url.strip())
if media_dict is not None:
existing_media = Media.get_by_id(
Media.render_key_name(media_dict['media_type_enum'],
media_dict['foreign_key']))
if existing_media is None \
or request.reference_key not in [reference.id() for reference in existing_media.references]:
media_dict['year'] = request.year
media_dict['reference_type'] = request.reference_type
media_dict['reference_key'] = request.reference_key
# Need to split deletehash out into its own private dict. Don't want that to be exposed via API...
if request.details_json:
incoming_details = json.loads(request.details_json)
private_details = None
if 'deletehash' in incoming_details:
private_details = {
'deletehash': incoming_details.pop('deletehash')
}
media_dict['private_details_json'] = json.dumps(
private_details) if private_details else None
media_dict['details_json'] = json.dumps(incoming_details)
suggestion = Suggestion(author=ndb.Key(Account, user_id),
target_model="media")
suggestion.contents = media_dict
suggestion.put()
return BaseResponse(code=200, message="Suggestion added")
else:
return BaseResponse(code=304,
message="Suggestion already exists")
else:
return BaseResponse(code=400, message="Bad suggestion url")
def _render(self, *args, **kw):
avatars_future = Media.query(Media.media_type_enum == MediaType.AVATAR).fetch_async()
avatars = sorted(avatars_future.get_result(), key=lambda a: int(a.references[0].id()[3:]))
self.template_values.update({
'avatars': avatars,
})
return jinja2_engine.render('avatars2018.html', self.template_values)
def get(self):
team_key = self.request.get("team_key")
year_str = self.request.get("year")
self._require_registration()
if not team_key or not year_str:
self.redirect("/", abort=True)
year = int(year_str)
team_future = Team.get_by_id_async(self.request.get("team_key"))
team = team_future.get_result()
if not team:
self.redirect("/", abort=True)
media_key_futures = Media.query(Media.references == team.key, Media.year == year).fetch_async(500, keys_only=True)
media_futures = ndb.get_multi_async(media_key_futures.get_result())
medias = [media_future.get_result() for media_future in media_futures]
medias_by_slugname = MediaHelper.group_by_slugname(medias)
self.template_values.update({
"status": self.request.get("status"),
"team": team,
"year": year,
"medias_by_slugname": medias_by_slugname,
})
self.response.out.write(jinja2_engine.render('suggestions/suggest_team_media.html', self.template_values))
def get(self):
team_key = self.request.get("team_key")
self._require_registration()
if not team_key:
self.redirect("/", abort=True)
team_future = Team.get_by_id_async(self.request.get("team_key"))
team = team_future.get_result()
if not team:
self.redirect("/", abort=True)
media_key_futures = Media.query(Media.references == team.key, Media.year == None).fetch_async(500, keys_only=True)
media_futures = ndb.get_multi_async(media_key_futures.get_result())
medias = [media_future.get_result() for media_future in media_futures]
social_medias = MediaHelper.get_socials(medias)
self.template_values.update({
"status": self.request.get("status"),
"team": team,
"social_medias": social_medias,
})
self.response.out.write(jinja2_engine.render('suggestions/suggest_team_social_media.html', self.template_values))
def _render(self, year):
year = int(year)
avatars = []
shards = memcache.get_multi(['{}avatars_{}'.format(year, i) for i in xrange(10)])
if len(shards) == 10: # If missing a shard, must refetch all
for _, shard in sorted(shards.items(), key=lambda kv: kv[0]):
avatars += shard
if not avatars:
avatars_future = Media.query(Media.media_type_enum == MediaType.AVATAR, Media.year == year).fetch_async()
avatars = sorted(avatars_future.get_result(), key=lambda a: int(a.references[0].id()[3:]))
shards = {}
size = len(avatars) / 10 + 1
for i in xrange(10):
start = i * size
end = start + size
shards['{}avatars_{}'.format(year, i)] = avatars[start:end]
memcache.set_multi(shards, 60*60*24)
self.template_values.update({
'year': year,
'avatars': avatars,
})
return jinja2_engine.render('avatars.html', self.template_values)
def get(self, event_key):
self._require_admin()
event = Event.get_by_id(event_key)
if not event:
self.abort(404)
event.prepAwardsMatchesTeams()
reg_sitevar = Sitevar.get_by_id("cmp_registration_hacks")
api_keys = ApiAuthAccess.query(ApiAuthAccess.event_list == ndb.Key(Event, event_key)).fetch()
event_medias = Media.query(Media.references == event.key).fetch(500)
self.template_values.update({
"event": event,
"medias": event_medias,
"cache_key": event_controller.EventDetail('2016nyny').cache_key.format(event.key_name),
"flushed": self.request.get("flushed"),
"playoff_types": PlayoffType.type_names,
"write_auths": api_keys,
"event_sync_disable": reg_sitevar and event_key in reg_sitevar.contents.get('divisions_to_skip', []),
"set_start_day_to_last": reg_sitevar and event_key in reg_sitevar.contents.get('set_start_to_last_day', []),
"skip_eventteams": reg_sitevar and event_key in reg_sitevar.contents.get('skip_eventteams', []),
"event_name_override": next(iter(filter(lambda e: e.get("event") == event_key, reg_sitevar.contents.get("event_name_override", []))), {}).get("name", "")
})
path = os.path.join(os.path.dirname(__file__), '../../templates/admin/event_details.html')
self.response.out.write(template.render(path, self.template_values))
def get(self, team_number):
self._require_admin()
team = Team.get_by_id("frc" + team_number)
event_teams = EventTeam.query(EventTeam.team == team.key).fetch(500)
team_medias = Media.query(Media.references == team.key).fetch(500)
robots = Robot.query(Robot.team == team.key).fetch()
district_teams = DistrictTeam.query(DistrictTeam.team == team.key).fetch()
team_medias_by_year = {}
for media in team_medias:
if media.year in team_medias_by_year:
team_medias_by_year[media.year].append(media)
else:
team_medias_by_year[media.year] = [media]
self.template_values.update({
'event_teams': event_teams,
'team': team,
'team_medias_by_year': team_medias_by_year,
'robots': robots,
'district_teams': district_teams,
})
path = os.path.join(os.path.dirname(__file__), '../../templates/admin/team_details.html')
self.response.out.write(template.render(path, self.template_values))
def get(self):
super(SuggestDesignsReviewController, self).get()
if self.request.get('action') and self.request.get('id'):
# Fast-path review
self._fastpath_review()
suggestions = Suggestion.query().filter(
Suggestion.review_state == Suggestion.REVIEW_PENDING).filter(
Suggestion.target_model == "robot").fetch(limit=50)
reference_keys = []
for suggestion in suggestions:
reference_key = suggestion.contents['reference_key']
reference = Media.create_reference(
suggestion.contents['reference_type'],
reference_key)
reference_keys.append(reference)
reference_futures = ndb.get_multi_async(reference_keys)
references = map(lambda r: r.get_result(), reference_futures)
suggestions_and_references = zip(suggestions, references)
self.template_values.update({
"suggestions_and_references": suggestions_and_references,
})
self.response.out.write(jinja2_engine.render('suggestions/suggest_designs_review.html', self.template_values))
def get(self):
team_key = self.request.get("team_key")
year_str = self.request.get("year")
self._require_login("/suggest/team/media?team_key=%s&year=%s" % (team_key, year_str))
if not team_key or not year_str:
self.redirect("/", abort=True)
year = int(year_str)
team_future = Team.get_by_id_async(self.request.get("team_key"))
team = team_future.get_result()
media_key_futures = Media.query(Media.references == team.key, Media.year == year).fetch_async(500, keys_only=True)
media_futures = ndb.get_multi_async(media_key_futures.get_result())
medias_by_slugname = MediaHelper.group_by_slugname([media_future.get_result() for media_future in media_futures])
self.template_values.update({
"success": self.request.get("success"),
"team": team,
"year": year,
"medias_by_slugname": medias_by_slugname
})
path = os.path.join(os.path.dirname(__file__), '../../templates/suggest_team_media.html')
self.response.out.write(template.render(path, self.template_values))
def _render(self, team_key, year=None):
self._set_team(team_key)
media_keys = Media.query(Media.references == self.team.key, Media.year == self.year).fetch(500, keys_only=True)
medias = ndb.get_multi(media_keys)
media_list = [ModelToDict.mediaConverter(media) for media in medias]
return json.dumps(media_list, ensure_ascii=True)
def get(self, team_number):
self._require_admin()
team = Team.get_by_id("frc" + team_number)
if not team:
self.abort(404)
event_teams = EventTeam.query(EventTeam.team == team.key).fetch(500)
team_medias = Media.query(Media.references == team.key).fetch(500)
robots = Robot.query(Robot.team == team.key).fetch()
district_teams = DistrictTeam.query(DistrictTeam.team == team.key).fetch()
years_participated = sorted(TeamParticipationQuery(team.key_name).fetch())
team_medias_by_year = {}
for media in team_medias:
if media.year in team_medias_by_year:
team_medias_by_year[media.year].append(media)
else:
team_medias_by_year[media.year] = [media]
media_years = sorted(team_medias_by_year.keys(), reverse=True)
self.template_values.update({
'event_teams': event_teams,
'team': team,
'team_media_years': media_years,
'team_medias_by_year': team_medias_by_year,
'robots': robots,
'district_teams': district_teams,
'years_participated': years_participated,
})
path = os.path.join(os.path.dirname(__file__), '../../templates/admin/team_details.html')
self.response.out.write(template.render(path, self.template_values))
def from_suggestion(cls, suggestion):
team_reference = Media.create_reference(
suggestion.contents['reference_type'],
suggestion.contents['reference_key'])
media = MediaCreator.create_media_model(suggestion, team_reference)
return MediaManipulator.createOrUpdate(media)
def parse(self, response):
"""
Parse team avatar from FMSAPI
"""
# Get team avatar
current_page = response['pageCurrent']
total_pages = response['pageTotal']
teams = response['teams']
avatars = []
keys_to_delete = set()
for teamData in teams:
team_number = teamData['teamNumber']
foreign_key = "avatar_{}_frc{}".format(self.year, team_number)
media_key = ndb.Key('Media', Media.render_key_name(MediaType.AVATAR, foreign_key))
if not teamData['encodedAvatar']:
keys_to_delete.add(media_key)
continue
else:
encoded_avatar = teamData['encodedAvatar']
avatar = Media(
key=media_key,
details_json=json.dumps({'base64Image': encoded_avatar}),
foreign_key=foreign_key,
media_type_enum=MediaType.AVATAR,
references=[ndb.Key('Team', "frc{}".format(team_number))],
year=self.year
)
avatars.append(avatar)
return (avatars, keys_to_delete, (current_page < total_pages))
def get(self):
suggestions = Suggestion.query().filter(
Suggestion.review_state == Suggestion.REVIEW_PENDING).filter(
Suggestion.target_model == "event_media").fetch(limit=50)
# Quick and dirty way to group images together
suggestions = sorted(suggestions, key=lambda x: 0 if x.contents['media_type_enum'] in MediaType.image_types else 1)
reference_keys = []
for suggestion in suggestions:
reference_key = suggestion.contents['reference_key']
reference = Media.create_reference(
suggestion.contents['reference_type'],
reference_key)
reference_keys.append(reference)
if 'details_json' in suggestion.contents:
suggestion.details = json.loads(suggestion.contents['details_json'])
if 'image_partial' in suggestion.details:
suggestion.details['thumbnail'] = suggestion.details['image_partial'].replace('_l', '_m')
reference_futures = ndb.get_multi_async(reference_keys)
references = map(lambda r: r.get_result(), reference_futures)
suggestions_and_references = zip(suggestions, references)
self.template_values.update({
"suggestions_and_references": suggestions_and_references,
})
self.response.out.write(jinja2_engine.render('suggestions/suggest_event_media_review_list.html', self.template_values))
def _query_async(self):
team_key = self._query_args[0]
year = self._query_args[1]
medias = yield Media.query(
Media.references == ndb.Key(Team, team_key),
Media.year == year).fetch_async()
raise ndb.Return(medias)
def get(self):
team_key = self.request.get("team_key")
year_str = self.request.get("year")
self._require_registration()
if not team_key or not year_str:
self.redirect("/", abort=True)
year = int(year_str)
team_future = Team.get_by_id_async(self.request.get("team_key"))
team = team_future.get_result()
if not team:
self.redirect("/", abort=True)
media_key_futures = Media.query(Media.references == team.key, Media.year == year).fetch_async(500, keys_only=True)
social_media_future = media_query.TeamSocialMediaQuery(team.key.id()).fetch_async()
media_futures = ndb.get_multi_async(media_key_futures.get_result())
medias = [media_future.get_result() for media_future in media_futures]
medias_by_slugname = MediaHelper.group_by_slugname(medias)
social_medias = sorted(social_media_future.get_result(), key=MediaHelper.social_media_sorter)
social_medias = filter(lambda m: m.media_type_enum == MediaType.INSTAGRAM_PROFILE, social_medias) # we only allow IG media, so only show IG profile
self.template_values.update({
"medias_by_slugname": medias_by_slugname,
"social_medias": social_medias,
"status": self.request.get("status"),
"team": team,
"year": year,
})
self.response.out.write(jinja2_engine.render('suggestions/suggest_team_media.html', self.template_values))
def get(self):
team_keys_future = Team.query().order(Team.team_number).fetch_async(keys_only=True)
social_media_keys_future = Media.query(Media.year == None).fetch_async(keys_only=True)
team_futures = ndb.get_multi_async(team_keys_future.get_result())
social_futures = ndb.get_multi_async(social_media_keys_future.get_result())
socials_by_team = defaultdict(dict)
for social_future in social_futures:
social = social_future.get_result()
for reference in social.references:
socials_by_team[reference.id()][social.media_type_enum] = social
if team_futures:
with cloudstorage.open(self.TEAMS_FILENAME_PATTERN, 'w') as teams_file:
writer = csv.writer(teams_file, delimiter=',')
for team_future in team_futures:
team = team_future.get_result()
team_row = [team.key.id(), team.nickname, team.name, team.city, team.state_prov, team.country, team.website, team.rookie_year]
for social_type in MediaType.social_types:
social = socials_by_team[team.key.id()].get(social_type, None)
team_row.append(social.social_profile_url if social is not None else None)
self._writerow_unicode(writer, team_row)
self.response.out.write("Done backing up teams!")
def get(self):
suggestions = Suggestion.query().filter(
Suggestion.review_state == Suggestion.REVIEW_PENDING).filter(
Suggestion.target_model == "media")
reference_keys = []
for suggestion in suggestions:
reference_keys.append(Media.create_reference(
suggestion.contents['reference_type'],
suggestion.contents['reference_key']))
if 'details_json' in suggestion.contents:
suggestion.details = json.loads(suggestion.contents['details_json'])
if 'image_partial' in suggestion.details:
suggestion.details['thumbnail'] = suggestion.details['image_partial'].replace('_l', '_m')
reference_futures = ndb.get_multi_async(reference_keys)
references = map(lambda r: r.get_result(), reference_futures)
suggestions_and_references = zip(suggestions, references)
self.template_values.update({
"suggestions_and_references": suggestions_and_references,
})
path = os.path.join(os.path.dirname(__file__), '../../templates/suggest_team_media_review_list.html')
self.response.out.write(template.render(path, self.template_values))
