class Election(db.Model):
"""
An election that users may vote for.
"""
name = db.StringProperty(required=True)
start = db.DateTimeProperty(required=True) # Time when voting begins
end = db.DateTimeProperty(required=True) # Time when voting ends
organization = db.ReferenceProperty(Organization,
collection_name='elections')
result_computed = db.BooleanProperty(required=True, default=False)
result_delay = db.IntegerProperty(
required=True, default=0) # Results delay to public in seconds
universal = db.BooleanProperty(required=True, default=False)
hidden = db.BooleanProperty(required=True, default=False)
task_count = db.IntegerProperty(
required=True, # Counter used to identify tasks
default=0)
voter_count = db.IntegerProperty(required=True, default=0)
voted_count = db.IntegerProperty(required=True, default=0)
description = db.TextProperty(required=False, default="")
def to_json(self, parseable_date=False):
# calculate the publication date by adding result delay to end
pub = self.end + timedelta(seconds=self.result_delay)
# parseable_date means it can be parsed in JS with Date.parse()
times = {
'start': self.start.strftime('%a, %B %d, %Y, %I:%M %p') + ' UTC',
'end': self.end.strftime('%a, %B %d, %Y, %I:%M %p') + ' UTC',
'pub': pub.strftime('%a, %B %d, %Y, %I:%M %p') + ' UTC'
} if parseable_date else {
'start': str(self.start),
'end': str(self.end),
'pub': str(pub)
}
now = datetime.now()
status = 'Not started'
if now > self.start:
status = 'Voting in progress'
if now > self.end:
status = 'Voting has ended'
if self.result_computed:
status = 'Result computed'
return {
'id': str(self.key()),
'name': self.name,
'organization': self.organization.name,
'times': times,
'result_computed': self.result_computed,
'result_delay': self.result_delay,
'universal': self.universal,
'hidden': self.hidden,
'voter_count': self.voter_count,
'voted_count': self.voted_count,
'description': self.description,
'status': status,
}
@property
def election_positions(self):
return [
ep for ep in ElectionPosition.gql(
"WHERE election=:1 ORDER BY datetime_created", self)
]
class Artwork(db.Model):
"""An artwork with a creation date."""
version = db.IntegerProperty(required=True)
data = db.TextProperty(required=True)
date = db.DateTimeProperty(auto_now_add=True)
class Like(db.Model):
blog_id = db.IntegerProperty(required=True)
viewer = db.StringProperty(required=True)
class User(db.Model): name = db.StringProperty(required = True) room = db.StringProperty(required = True) role = db.IntegerProperty() is_alive = db.BooleanProperty()
class Episode(db.Model):
show = db.ReferenceProperty(Show)
season = db.ReferenceProperty(Season)
season_number = db.IntegerProperty()
number = db.IntegerProperty()
title = db.StringProperty()
text = db.TextProperty(default="")
date = db.DateTimeProperty()
_memkey_episode_dict = "all_episodes_dict"
@classmethod
def kind(cls):
return "series_episode"
@property
def date_end(self):
return self.date + datetime.timedelta(minutes=self.show.runtime)
@property
def date_local(self):
if getattr(self, "_date_local", None) is None:
try:
tz = get_timezone_for_gmt_offset(self.show.timezone)
except Exception:
tz = utc
self._date_local = utc.localize(self.date).astimezone(tz)
return self._date_local
@property
def date_local_end(self):
if getattr(self, "_date_local_end", None) is None:
try:
tz = get_timezone_for_gmt_offset(self.show.timezone)
except Exception:
tz = utc
self._date_local_end = utc.localize(self.date_end).astimezone(tz)
return self._date_local_end
@classmethod
def update_or_create(cls, season, episode_info):
episode = Episode.all().filter("show =", season.show).filter(
"season =", season).filter("number =", episode_info.nr).get()
logging.debug("Found episode... %s" % episode)
if episode is None:
episode = Episode.create(season, episode_info)
else:
episode.update(episode_info)
episode.put()
return episode
@classmethod
def create(cls, season, episode_info):
return Episode(show=season.show,
season=season,
season_number=season.number,
number=episode_info.nr,
title=episode_info.title,
date=episode_info.date)
@classmethod
def get_all_dict(cls):
episode_dict = memcache.get(cls._memkey_episode_dict)
if episode_dict is not None:
return episode_dict
now = datetime.datetime.now()
one_week_ago = now - datetime.timedelta(days=8)
# in_one_week = now + datetime.timedelta(days=8)
episodes = Episode.all().filter("date >", one_week_ago)
# removed this: .filter("date <",in_one_week).fetch(1000)
episode_dict = {}
for ep in episodes:
if len(episode_dict.get(str(ep._show), [])) < 20:
# store max of 20 episodes per show
episode_dict.setdefault(str(ep._show), []).append(ep)
memcache.set(key=cls._memkey_episode_dict, value=episode_dict)
return episode_dict
@classmethod
def clear_cache(cls):
memcache.delete(cls._memkey_episode_dict)
@classmethod
def add_clear_cache_task(cls, queue_name):
t = taskqueue.Task(url=reverse('seriesly-shows-clear_cache'),
params={})
t.add(queue_name=queue_name)
return t
@classmethod
def get_for_shows(cls, shows, before=None, after=None, order=None):
episode_list = []
episode_dict = Episode.get_all_dict()
changed = False
for show in shows:
k = str(show.key())
if k in episode_dict:
episode_dict[k].sort(key=lambda x: x.date)
prev = None
for ep in episode_dict[k]:
if prev is not None:
prev.next = ep
ep.show = show
prev = ep
episode_list.extend(episode_dict[k])
if changed:
memcache.set(key=cls._memkey_episode_dict, value=episode_dict)
episode_list.sort(key=lambda x: x.date)
if after is not None or before is not None:
lower = None
upper = len(episode_list)
for ep, i in zip(episode_list, range(len(episode_list))):
if after is not None and lower is None and ep.date > after:
lower = i
if before is not None and ep.date > before:
upper = i
break
if lower > 0 or upper < len(episode_list):
episode_list = episode_list[lower:upper]
if order is not None and order.startswith("-"):
episode_list.reverse()
return episode_list
@classmethod
def get_for_shows_old(cls, shows, before=None, after=None, order=None):
def extra(q):
if before is not None:
q = q.filter("date <", before)
if after is not None:
q = q.filter("date >", after)
if order is not None:
q = q.order(order)
return q
if not len(shows):
return []
if len(shows) <= 28:
logging.debug("starting query")
query = Episode.all().filter("show IN", shows)
return extra(query).fetch(1000)
episodes = []
for i in range(len(shows) / 28 + 1):
q_shows = shows[i * 28:(i + 1) * 28]
if not len(q_shows):
continue
episodes.extend(
extra(Episode.all().filter("show IN", q_shows)).fetch(1000))
if order is not None and order.startswith("-"):
return sorted(episodes, lambda x: x.date).reverse()
else:
return sorted(episodes, lambda x: x.date)
def update(self, episode_info):
self.title = episode_info.title
self.date = episode_info.date
def get_next(self):
return Episode.all().filter("date >", self.date).get()
def create_event_details(self, cal):
vevent = cal.add('vevent')
vevent.add('uid').value = "seriesly-episode-%s" % self.key()
try:
tz = get_timezone_for_gmt_offset(self.show.timezone)
except Exception:
tz = utc
date = utc.localize(self.date).astimezone(tz)
vevent.add('dtstart').value = date
vevent.add('dtend').value = date + datetime.timedelta(
minutes=self.show.runtime)
vevent.add('summary').value = "%s - %s (%dx%d)" % (
self.show.name, self.title, self.season_number, self.number)
vevent.add('location').value = self.show.network
return vevent
class Tags(db.Model):
'''Model to collect tags with count of using every tag.'''
name = db.StringProperty()
count = db.IntegerProperty()
class SpamCounts(db.Model):
nham = db.IntegerProperty()
nspam = db.IntegerProperty()
class UserData(db.Model):
user = db.UserProperty()
moderator = db.BooleanProperty(default=False)
joined = db.DateTimeProperty(auto_now_add=True)
last_login = db.DateTimeProperty()
proficient_exercises = db.StringListProperty(
) # Names of exercises in which the user is *explicitly* proficient
all_proficient_exercises = db.StringListProperty(
) # Names of all exercises in which the user is proficient
suggested_exercises = db.StringListProperty()
assigned_exercises = db.StringListProperty()
badges = db.StringListProperty() # All awarded badges
need_to_reassess = db.BooleanProperty()
points = db.IntegerProperty()
total_seconds_watched = db.IntegerProperty(default=0)
coaches = db.StringListProperty()
map_coords = db.StringProperty()
expanded_all_exercises = db.BooleanProperty(default=True)
videos_completed = db.IntegerProperty(default=-1)
last_daily_summary = db.DateTimeProperty()
last_activity = db.DateTimeProperty()
count_feedback_notification = db.IntegerProperty(default=-1)
@staticmethod
def get_for_current_user():
user = util.get_current_user()
if user is not None:
user_data = UserData.get_for(user)
if user_data is not None:
return user_data
return UserData()
@staticmethod
def get_for(user):
query = UserData.all()
query.filter('user ='******'-points') # Temporary workaround for issue 289
return query.get()
@staticmethod
def get_or_insert_for(user):
# Once we have rekeyed legacy entities,
# the next block can just be a call to .get_or_insert()
user_data = UserData.get_for(user)
if user_data is None:
if user.email():
key = "user_email_key_%s" % user.email()
user_data = UserData.get_or_insert(
key_name=key,
user=user,
moderator=False,
last_login=datetime.datetime.now(),
proficient_exercises=[],
suggested_exercises=[],
assigned_exercises=[],
need_to_reassess=True,
points=0,
coaches=[])
return user_data
def get_or_insert_exercise(self, exercise, allow_insert=True):
exid = exercise.name
userExercise = UserExercise.get_by_key_name(exid, parent=self)
if not userExercise:
# There are some old entities lying around that don't have keys.
# We have to check for them here, but once we have reparented and rekeyed legacy entities,
# this entire function can just be a call to .get_or_insert()
query = UserExercise.all()
query.filter('user ='******'exercise =', exid)
query.order('-total_done') # Temporary workaround for issue 289
userExercise = query.get()
if allow_insert and not userExercise:
userExercise = UserExercise.get_or_insert(
key_name=exid,
parent=self,
user=self.user,
exercise=exid,
exercise_model=exercise,
streak=0,
longest_streak=0,
first_done=datetime.datetime.now(),
last_done=datetime.datetime.now(),
total_done=0,
)
return userExercise
def reassess_from_graph(self, ex_graph):
all_proficient_exercises = []
for ex in ex_graph.get_proficient_exercises():
all_proficient_exercises.append(ex.name)
suggested_exercises = []
for ex in ex_graph.get_suggested_exercises():
suggested_exercises.append(ex.name)
is_changed = (all_proficient_exercises != self.all_proficient_exercises
or suggested_exercises != self.suggested_exercises)
self.all_proficient_exercises = all_proficient_exercises
self.suggested_exercises = suggested_exercises
self.need_to_reassess = False
return is_changed
def reassess_if_necessary(self, user=None):
if not self.need_to_reassess or self.all_proficient_exercises is None:
return False
ex_graph = ExerciseGraph(self, user)
return self.reassess_from_graph(ex_graph)
def is_proficient_at(self, exid, user=None):
self.reassess_if_necessary(user)
return (exid in self.all_proficient_exercises)
def is_explicitly_proficient_at(self, exid):
return (exid in self.proficient_exercises)
def is_reviewing(self, exid, user_exercise, time):
# Short circuit out of full review check if not proficient or review time hasn't come around yet
if not self.is_proficient_at(exid):
return False
if user_exercise.last_review + user_exercise.get_review_interval(
) > time:
return False
ex_graph = ExerciseGraph(self)
review_exercise_names = map(lambda exercise: exercise.name,
ex_graph.get_review_exercises(time))
return (exid in review_exercise_names)
def is_suggested(self, exid):
self.reassess_if_necessary()
return (exid in self.suggested_exercises)
def get_students_data(self):
coach_email = self.user.email()
query = db.GqlQuery("SELECT * FROM UserData WHERE coaches = :1",
coach_email)
students_data = []
for student_data in query:
students_data.append(student_data)
if coach_email.lower() != coach_email:
students_set = set(
map(lambda student_data: student_data.key().id_or_name(),
students_data))
query = db.GqlQuery("SELECT * FROM UserData WHERE coaches = :1",
coach_email.lower())
for student_data in query:
if student_data.key().id_or_name() not in students_set:
students_data.append(student_data)
return students_data
def get_students(self):
return map(lambda student_data: student_data.user.email(),
self.get_students_data())
def add_points(self, points):
if self.points == None:
self.points = 0
self.points += points
def get_videos_completed(self):
if self.videos_completed < 0:
self.videos_completed = UserVideo.count_completed_for_user(
self.user)
self.put()
return self.videos_completed
def feedback_notification_count(self):
if self.count_feedback_notification == -1:
self.count_feedback_notification = models_discussion.FeedbackNotification.gql(
"WHERE user = :1", self.user).count()
self.put()
return self.count_feedback_notification
class Video(Searchable, db.Model):
youtube_id = db.StringProperty()
url = db.StringProperty()
title = db.StringProperty()
description = db.TextProperty()
playlists = db.StringListProperty()
keywords = db.StringProperty()
duration = db.IntegerProperty(default=0)
# Human readable, unique id that can be used in URLS.
readable_id = db.StringProperty()
# YouTube view count from last sync.
views = db.IntegerProperty(default=0)
# Date first added via KA library sync with YouTube.
# This property hasn't always existsed, so for many old videos
# this date may be much later than the actual YouTube upload date.
date_added = db.DateTimeProperty(auto_now_add=True)
INDEX_ONLY = ['title', 'keywords', 'description']
INDEX_TITLE_FROM_PROP = 'title'
INDEX_USES_MULTI_ENTITIES = False
@staticmethod
def get_for_readable_id(readable_id):
video = None
query = Video.all()
query.filter('readable_id =', readable_id)
# The following should just be:
# video = query.get()
# but the database currently contains multiple Video objects for a particular
# video. Some are old. Some are due to a YouTube sync where the youtube urls
# changed and our code was producing youtube_ids that ended with '_player'.
# This hack gets the most recent valid Video object.
key_id = 0
for v in query:
if v.key().id() > key_id and not v.youtube_id.endswith('_player'):
video = v
key_id = v.key().id()
# End of hack
return video
def first_playlist(self):
query = VideoPlaylist.all()
query.filter('video =', self)
query.filter('live_association =', True)
video_playlist = query.get()
if video_playlist:
return video_playlist.playlist
return None
def current_user_points(self):
user_video = UserVideo.get_for_video_and_user(self,
util.get_current_user())
if user_video:
return points.VideoPointCalculator(user_video)
else:
return 0
@staticmethod
def get_dict(query, fxn_key):
video_dict = {}
for video in query.fetch(10000):
video_dict[fxn_key(video)] = video
return video_dict
class LikedUser(db.Model):
userid = db.IntegerProperty(required=True)
postid = db.IntegerProperty(required=True)
class UserExercise(db.Model):
user = db.UserProperty()
exercise = db.StringProperty()
exercise_model = db.ReferenceProperty(Exercise)
streak = db.IntegerProperty(default=0)
longest_streak = db.IntegerProperty(default=0)
first_done = db.DateTimeProperty(auto_now_add=True)
last_done = db.DateTimeProperty()
total_done = db.IntegerProperty(default=0)
last_review = db.DateTimeProperty(default=datetime.datetime.min)
review_interval_secs = db.IntegerProperty(default=(
60 * 60 * 24 *
consts.DEFAULT_REVIEW_INTERVAL_DAYS)) # Default 7 days until review
proficient_date = db.DateTimeProperty()
seconds_per_fast_problem = db.FloatProperty(
default=consts.MIN_SECONDS_PER_FAST_PROBLEM
) # Seconds expected to finish a problem 'quickly' for badge calculation
summative = db.BooleanProperty(default=False)
_USER_EXERCISE_KEY_FORMAT = "UserExercise.all().filter('user = '******')"
@staticmethod
def get_key_for_user(user):
return UserExercise._USER_EXERCISE_KEY_FORMAT % user.email()
@staticmethod
def get_for_user_use_cache(user):
user_exercises_key = UserExercise.get_key_for_user(user)
user_exercises = memcache.get(user_exercises_key)
if user_exercises is None:
query = UserExercise.all()
query.filter('user ='******'t schedule a review
if (self.streak + correct) < self.required_streak(
) and self.longest_streak < self.required_streak():
return
# If the user is hitting a new streak either for the first time or after having lost
# proficiency, reset their review interval counter.
if (self.streak + correct) >= self.required_streak:
self.review_interval_secs = 60 * 60 * 24 * consts.DEFAULT_REVIEW_INTERVAL_DAYS
review_interval = self.get_review_interval()
if correct and self.last_review != datetime.datetime.min:
time_since_last_review = now - self.last_review
if time_since_last_review >= review_interval:
review_interval = time_since_last_review * 2
if not correct:
review_interval = review_interval // 2
if correct:
self.last_review = now
else:
self.last_review = datetime.datetime.min
self.review_interval_secs = review_interval.days * 86400 + review_interval.seconds
def set_proficient(self, proficient, user_data):
if not proficient and self.longest_streak < self.required_streak():
# Not proficient and never has been so nothing to do
return
if proficient:
if self.exercise not in user_data.proficient_exercises:
user_data.proficient_exercises.append(self.exercise)
user_data.need_to_reassess = True
user_data.put()
else:
if self.exercise in user_data.proficient_exercises:
user_data.proficient_exercises.remove(self.exercise)
user_data.need_to_reassess = True
user_data.put()
class Issue(db.Model):
"""Represents a single issue which is being voted on"""
status = db.StringProperty(required=True,
default='active',
choices=set(['active', 'done', 'canceled']))
is_public = db.BooleanProperty(
default=False) #is issue listed on site or just by sharing url?
creator = db.UserProperty(auto_current_user=True)
title = db.StringProperty(required=True)
description = db.TextProperty()
duration = db.IntegerProperty()
creation_date = db.DateTimeProperty(auto_now_add=True)
start_time = db.DateTimeProperty() #time when first vote is cast
end_time = db.DateTimeProperty() #time when vote will end
#Implicit Properties:
#choices = Implicitly created list of choice objects
#votes - implicitly created list of vote objects
def add_choice(self, choice_name):
new_choice = Choice(name=choice_name, issue=self)
new_choice.put()
def remove_choice(self, choice):
choice.delete()
def vote_count(self):
return self.votes.count(999999)
def vote_for_member(self, member=None):
if not member:
member = users.get_current_user()
logging.info('member:%s voted:%s' %
(member.nickname(), self.votes.filter('member =',
member).fetch(20)))
return self.votes.filter('member =', member).get()
def register_vote(self, choice, member=None):
if not member:
member = users.get_current_user()
member_vote = self.vote_for_member(member)
was_changed = False
if (member_vote):
member_vote.choice = choice
was_changed = True
else:
member_vote = Vote(member=member, choice=choice, issue=self)
member_vote.put()
if (not self.start_time):
self.start_time = datetime.now()
self.end_time = self.start_time + timedelta(hours=self.duration)
self.put()
return was_changed
def extend_duration(self, hours):
self.duration += hours
if self.start_time:
self.end_time = self.start_time + timedelta(hours=self.duration)
self.put()
def days_left(self):
delta = self.end_time - datetime.now()
return delta.days
def hours_left(self):
delta = self.end_time - datetime.now()
#days = delta.days
#hours = days*24 + delta.seconds/3600
hours = delta.seconds / 3600
return hours
def is_active(self):
return self.status in ('active')
def has_results(self):
return self.status in ('done')
def member_is_creator(self, member=None):
if not member:
member = users.get_current_user()
return member == self.creator
def winning_choices(
self): #returns list of keys of winning choices (may be a tie)
result = []
high_vote = 0
for choice in self.choices:
cnt = choice.vote_count()
if cnt == high_vote:
result.append(choice.key())
elif cnt > high_vote:
result = [choice.key()]
high_vote = cnt
return result
def update_status(self):
if self.is_active:
if self.end_time:
if self.end_time <= datetime.now():
logging.info('status changed for issue: %s' % (self.title))
self.status = 'done'
self.put()
@classmethod
def issues_created_by(cls, member=None, limit=20):
if not member:
member = users.get_current_user()
return cls.all().filter('creator =',
member).order('-creation_date').fetch(limit)
@classmethod
def issues_voted_on(cls, member=None, limit=20):
if not member:
member = users.get_current_user()
if not member: #if logged out
return []
member_votes = Vote.all().filter(
'member =', member).order('-update_time').fetch(limit)
##logging.info('member_votes:%s' % (member_votes))
##logging.info('output:%s' % ([vote.issue for vote in member_votes]))
return [vote.issue for vote in member_votes]
@classmethod
def recent_results(
cls,
member=None,
limit=20
): #*** Need to fix, limit will be incorrect here because of filtering
if not member:
member = users.get_current_user()
if not member: #if logged out
return []
recent = cls.all().filter('status =',
'done').order('-end_time').fetch(limit)
return [issue for issue in recent
if issue.vote_for_member()] #***this is probably slow
class Counter(db.Model):
type = db.StringProperty(multiline=False)
period_start_time = db.DateTimeProperty()
count = db.IntegerProperty()
class Membership(db.Model):
hash = db.StringProperty()
first_name = db.StringProperty(required=True)
last_name = db.StringProperty(required=True)
email = db.StringProperty(required=True)
# The hash of the user's password.
# TODO(danielp): Make this required after we finish migrating away from domain
# accounts.
password_hash = db.StringProperty()
twitter = db.StringProperty(required=False)
plan = db.StringProperty(required=False)
status = db.StringProperty() # None, active, suspended
referuserid = db.StringProperty()
referrer = db.StringProperty()
rfid_tag = db.StringProperty()
extra_599main = db.StringProperty()
extra_dnd = db.BooleanProperty(default=False)
auto_signin = db.StringProperty()
unsubscribe_reason = db.TextProperty()
hardship_comment = db.TextProperty()
spreedly_token = db.StringProperty()
parking_pass = db.StringProperty()
created = db.DateTimeProperty(auto_now_add=True)
updated = db.DateTimeProperty()
# How many times the user has signed in this month.
signins = db.IntegerProperty(default=0)
# When the last time they signed in was.
last_signin = db.DateTimeProperty()
# The following are legacy parameters.
# TODO(danielp): Remove these after we complete the migration away from
# domain accounts.
# Whether we've created a google apps user yet.
domain_user = db.BooleanProperty(default=False)
# The user's domain username.
username = db.StringProperty()
# Temporarily stores the user's domain password.
password = db.StringProperty(default=None)
""" Override of the default put method which allows us to skip changing the
updated property for testing purposes.
skip_time_update: Whether or not to set updated to the current date and time.
"""
def put(self, *args, **kwargs):
if not kwargs.pop("skip_time_update", False):
self.updated = datetime.datetime.now()
super(Membership, self).put(*args, **kwargs)
def icon(self):
return str("http://www.gravatar.com/avatar/" + hashlib.md5(self.email.lower()).hexdigest())
def full_name(self):
return '%s %s' % (self.first_name, self.last_name)
def spreedly_url(self):
config = Config()
return str("https://subs.pinpayments.com/%s/subscriber_accounts/%s" % \
(config.SPREEDLY_ACCOUNT, self.spreedly_token))
def spreedly_admin_url(self):
config = Config()
return str("https://subs.pinpayments.com/%s/subscribers/%s" % \
(config.SPREEDLY_ACCOUNT, self.key().id()))
def subscribe_url(self, plan=None):
config = Config()
if not plan:
plan = self.plan
url = "https://subs.pinpayments.com/%s/subscribers/%i/%s/subscribe/%s" % \
(config.SPREEDLY_ACCOUNT, self.key().id(),
self.spreedly_token, plans.Plan.get_by_name(plan).plan_id)
return str(url)
""" URL we use to subscribe a person for the first time.
host: The first part of the return URL, e.g. signup.hackerdojo.com.
plan: Optionally specifies a different plan to use. """
def new_subscribe_url(self, host, plan=None):
config = Config()
if not plan:
plan = self.plan
query_str = urllib.urlencode({"first_name": self.first_name,
"last_name": self.last_name,
"email": self.email,
"return_url": "http://%s/success/%s" % \
(host, self.hash)})
url = "https://subs.pinpayments.com/%s/subscribers/%i/subscribe/%s/%s?%s" % \
(config.SPREEDLY_ACCOUNT, self.key().id(),
plans.Plan.get_by_name(plan).plan_id, self.username, query_str)
return str(url)
def force_full_subscribe_url(self):
config = Config()
url = "https://subs.pinpayments.com/%s/subscribers/%i/%s/subscribe/%s" % \
(config.SPREEDLY_ACCOUNT, self.key().id(),
self.spreedly_token, plans.newfull.plan_id)
return str(url)
def unsubscribe_url(self):
return "http://signup.hackerdojo.com/unsubscribe/%i" % (self.key().id())
""" Returns this user's unique ID, which can be an integer or string. """
def get_id(self):
return self.key().id()
""" Sets the user's password.
password: The password which will be hashed and stored. """
def set_password(self, password):
logging.debug("Setting password for user %s." % (self.email))
self.password_hash = security.generate_password_hash(password, length=12)
""" Creates a new authorization token for a given user ID.
user_id: User unique ID.
Returns: A string with the authorization token. """
@classmethod
def create_auth_token(cls, user_id):
token = UserToken(user_id, "auth")
token.save()
return token.token
""" Deletes a given authorization token.
user_id: User unique ID.
token: A string with the authorization token. """
@classmethod
def delete_auth_token(cls, user_id, token):
token = UserToken.verify(user_id, "auth", token)
if not token:
logging.warning("Delete: Ignoring bad token for %d." % (user_id))
return
token.delete()
""" Returns a Membership object based on a user ID and token.
user_id: The unique ID of the requesting user.
token: The token string to be verified.
Returns: A tuple (Membership, timestamp), with a Membership object and
the token timestamp, or (None, None) if both were not found. """
@classmethod
def get_by_auth_token(cls, user_id, token):
# First, check that the token is valid.
token = UserToken.verify(user_id, "auth", token)
if not token:
logging.warning("Bad token, not getting user %d." % (user_id))
return (None, None)
user = cls.get_by_id(user_id)
timestamp = token.timestamp
return (user, timestamp)
""" Gets the user with the specified login credentials.
email: The email of the user.
password: The password of the user.
Returns: Membership object if found. """
@classmethod
def get_by_auth_password(cls, email, password):
user = cls.get_by_email(email)
if not user:
raise auth.InvalidAuthIdError("No user with email '%s'." % (email))
if not security.check_password_hash(password, user.password_hash):
raise auth.InvalidPasswordError("Bad password for user '%s'." % (email))
return user
""" Gets the user with the specified email.
email: Either the normal email, or the hackerdojo.com email of the user.
Returns: The membership object corresponding to the user, or None if no user
was found. """
@classmethod
def get_by_email(cls, email):
# TODO(danielp): Remove code for dealing with hackerdojo.com emails after
# we've finished migrating away from domain accounts.
if "@hackerdojo.com" in email:
username = email.split("@")[0]
return cls.get_by_username(username)
return cls.all().filter('email =', email).get()
@classmethod
def get_by_hash(cls, hash):
return cls.all().filter('hash =', hash).get()
# This is a legacy method:
# TODO(danielp): Remove this after we migrate away from domain accounts.
@classmethod
def get_by_username(cls, username):
return cls.all().filter('username ='******'s email. This will be used as a unique ID.
password: The user's raw password. Will be hashed before saving, obviously.
other_properties: Keyword arguments specifying properties that will be
forwarded to the Membership constructor. All the other required properties
should be in here.
Returns: The created Membership entity. """
@classmethod
def create_user(cls, email, password, **other_properties):
logging.info("Creating user with email '%s', other properties: %s." % \
(email, other_properties))
password_hash = security.generate_password_hash(password, length=12)
member = cls(email=email, password_hash=password_hash, **other_properties)
member.put()
return member
class _PipelineRecord(db.Model):
"""Represents a Pipeline.
Properties:
class_path: Path of the Python class to use for this pipeline.
root_pipeline: The root of the whole workflow; set to itself this pipeline
is its own root.
fanned_out: List of child _PipelineRecords that were started when this
generator pipeline moved from WAITING to RUN.
start_time: For pipelines with no start _BarrierRecord, when this pipeline
was enqueued to run immediately.
finalized_time: When this pipeline moved from WAITING or RUN to DONE.
params: Serialized parameter dictionary.
status: The current status of the pipeline.
current_attempt: The current attempt (starting at 0) to run.
max_attempts: Maximum number of attempts (starting at 0) to run.
next_retry_time: ETA of the next retry attempt.
retry_message: Why the last attempt failed; None or empty if no message.
Root pipeline properties:
is_root_pipeline: This is a root pipeline.
abort_message: Why the whole pipeline was aborted; only saved on
root pipelines.
abort_requested: If an abort signal has been requested for this root
pipeline; only saved on root pipelines
"""
WAITING = 'waiting'
RUN = 'run'
DONE = 'done'
ABORTED = 'aborted'
class_path = db.StringProperty()
root_pipeline = db.SelfReferenceProperty(
collection_name='child_pipelines_set')
fanned_out = db.ListProperty(db.Key, indexed=False)
start_time = db.DateTimeProperty(indexed=True)
finalized_time = db.DateTimeProperty(indexed=False)
# One of these two will be set, depending on the size of the params.
params_text = db.TextProperty(name='params')
params_blob = blobstore.BlobReferenceProperty(name='params_blob',
indexed=False)
status = db.StringProperty(choices=(WAITING, RUN, DONE, ABORTED),
default=WAITING)
# Retry behavior
current_attempt = db.IntegerProperty(default=0, indexed=False)
max_attempts = db.IntegerProperty(default=1, indexed=False)
next_retry_time = db.DateTimeProperty(indexed=False)
retry_message = db.TextProperty()
# Root pipeline properties
is_root_pipeline = db.BooleanProperty()
abort_message = db.TextProperty()
abort_requested = db.BooleanProperty(indexed=False)
@classmethod
def kind(cls):
return '_AE_Pipeline_Record'
@property
def params(self):
"""Returns the dictionary of parameters for this Pipeline."""
if hasattr(self, '_params_decoded'):
return self._params_decoded
if self.params_blob is not None:
value_encoded = self.params_blob.open().read()
else:
value_encoded = self.params_text
value = simplejson.loads(value_encoded, cls=util.JsonDecoder)
if isinstance(value, dict):
kwargs = value.get('kwargs')
if kwargs:
adjusted_kwargs = {}
for arg_key, arg_value in kwargs.iteritems():
# Python only allows non-unicode strings as keyword arguments.
adjusted_kwargs[str(arg_key)] = arg_value
value['kwargs'] = adjusted_kwargs
self._params_decoded = value
return self._params_decoded
def truncated_copy(self):
"""Create a lightweight copy of the pipeline with the args truncated."""
return _LowMemoryPipelineRecord(self)
class Exercise(db.Model):
name = db.StringProperty()
short_display_name = db.StringProperty(default="")
prerequisites = db.StringListProperty()
covers = db.StringListProperty()
v_position = db.IntegerProperty()
h_position = db.IntegerProperty()
seconds_per_fast_problem = db.FloatProperty(
default=consts.MIN_SECONDS_PER_FAST_PROBLEM
) # Seconds expected to finish a problem 'quickly' for badge calculation
# True if this exercise is live and visible to all users.
# Non-live exercises are only visible to admins.
live = db.BooleanProperty(default=False)
# True if this exercise is a quasi-exercise generated by
# combining the content of other exercises
summative = db.BooleanProperty(default=False)
# Teachers contribute raw html with embedded CSS and JS
# and we sanitize it with Caja before displaying it to
# students.
author = db.UserProperty()
raw_html = db.TextProperty()
last_modified = db.DateTimeProperty()
safe_html = db.TextProperty()
safe_js = db.TextProperty()
last_sanitized = db.DateTimeProperty(default=datetime.datetime.min)
sanitizer_used = db.StringProperty()
@staticmethod
def get_by_name(name):
dict_exercises = Exercise.__get_dict_use_cache_unsafe__()
if dict_exercises.has_key(name):
if dict_exercises[name].is_visible_to_current_user():
return dict_exercises[name]
return None
@staticmethod
def to_display_name(name):
if name:
return name.replace('_', ' ').capitalize()
return ""
def display_name(self):
return Exercise.to_display_name(self.name)
@staticmethod
def to_short_name(name):
exercise = Exercise.get_by_name(name)
if exercise:
return exercise.short_name()
return ""
def short_name(self):
if self.short_display_name:
return self.short_display_name[:11]
return self.display_name()[:11]
def is_visible_to_current_user(self):
return self.live or users.is_current_user_admin()
def required_streak(self):
if self.summative:
return consts.REQUIRED_STREAK * len(self.prerequisites)
else:
return consts.REQUIRED_STREAK
def struggling_threshold(self):
return 3 * self.required_streak()
def summative_children(self):
if not self.summative:
return []
query = db.Query(Exercise)
query.filter("name IN ", self.prerequisites)
return query
def non_summative_exercise(self, problem_number):
if not self.summative:
return self
if len(self.prerequisites) <= 0:
raise Exception(
"Summative exercise '%s' does not include any other exercises"
% self.name)
# For now we just cycle through all of the included exercises in a summative exercise
index = int(problem_number) % len(self.prerequisites)
exid = self.prerequisites[index]
query = Exercise.all()
query.filter('name =', exid)
exercise = query.get()
if not exercise:
raise Exception("Unable to find included exercise")
if exercise.summative:
return exercise.non_summative_exercise(problem_number)
else:
return exercise
def related_videos(self):
exercise_videos = None
query = ExerciseVideo.all()
query.filter('exercise =', self.key())
return query
@layer_cache.cache_with_key_fxn(
lambda self: "related_videos_%s" % self.key(),
layer=layer_cache.SINGLE_LAYER_MEMCACHE_ONLY)
def related_videos_fetch(self):
exercise_videos = self.related_videos().fetch(10)
for exercise_video in exercise_videos:
exercise_video.video # Pre-cache video entity
return exercise_videos
_CURRENT_SANITIZER = "http://caja.appspot.com/"
def ensure_sanitized(self):
if self.last_sanitized >= self.last_modified and self.sanitizer_used == Exercise._CURRENT_SANITIZER:
return
cajoled = cajole.cajole(self.raw_html)
if 'error' in cajoled:
raise Exception(cajoled['html'])
self.safe_html = db.Text(cajoled['html'])
self.safe_js = db.Text(cajoled['js'])
self.last_sanitized = datetime.datetime.now()
self.sanitizer = Exercise._CURRENT_SANITIZER
self.put()
@classmethod
def all(cls, live_only=False):
query = super(Exercise, cls).all()
if live_only or not users.is_current_user_admin():
query.filter("live =", True)
return query
@classmethod
def all_unsafe(cls):
return super(Exercise, cls).all()
@staticmethod
def get_all_use_cache():
if users.is_current_user_admin():
return Exercise.__get_all_use_cache_unsafe__()
else:
return Exercise.__get_all_use_cache_safe__()
@staticmethod
@layer_cache.cache_with_key_fxn(
lambda *args, **kwargs: "all_exercises_unsafe_%s" % Setting.
cached_exercises_date())
def __get_all_use_cache_unsafe__():
query = Exercise.all_unsafe().order('h_position')
return query.fetch(200)
@staticmethod
@layer_cache.cache_with_key_fxn(
lambda *args, **kwargs: "all_exercises_safe_%s" % Setting.
cached_exercises_date())
def __get_all_use_cache_safe__():
query = Exercise.all(live_only=True).order('h_position')
return query.fetch(200)
@staticmethod
@layer_cache.cache_with_key_fxn(
lambda *args, **kwargs: "all_exercises_dict_unsafe_%s" % Setting.
cached_exercises_date())
def __get_dict_use_cache_unsafe__():
exercises = Exercise.__get_all_use_cache_unsafe__()
dict_exercises = {}
for exercise in exercises:
dict_exercises[exercise.name] = exercise
return dict_exercises
_EXERCISES_COUNT_KEY = "Exercise.count()"
@staticmethod
def get_count():
count = memcache.get(Exercise._EXERCISES_COUNT_KEY,
namespace=App.version)
if count is None:
count = Exercise.all().count()
memcache.set(Exercise._EXERCISES_COUNT_KEY,
count,
namespace=App.version)
return count
def put(self):
Setting.cached_exercises_date(str(datetime.datetime.now()))
memcache.delete(Exercise._EXERCISES_COUNT_KEY, namespace=App.version)
db.Model.put(self)
class Settings(db.Model):
days_for_old_picture = db.IntegerProperty()
width_for_cover_album = db.IntegerProperty()
width_for_small_picture = db.IntegerProperty()
class VideoPlaylist(db.Model):
playlist = db.ReferenceProperty(Playlist)
video = db.ReferenceProperty(Video)
video_position = db.IntegerProperty()
# Lets us enable/disable video playlist relationships in bulk without removing the entry
live_association = db.BooleanProperty(default=False)
last_live_association_generation = db.IntegerProperty(default=0)
_VIDEO_PLAYLIST_KEY_FORMAT = "VideoPlaylist_Videos_for_Playlist_%s"
_PLAYLIST_VIDEO_KEY_FORMAT = "VideoPlaylist_Playlists_for_Video_%s"
@staticmethod
def get_cached_videos_for_playlist(playlist, limit=500):
key = VideoPlaylist._VIDEO_PLAYLIST_KEY_FORMAT % playlist.key()
namespace = str(App.version) + "_" + str(
Setting.cached_library_content_date())
videos = memcache.get(key, namespace=namespace)
if videos is None:
videos = []
query = VideoPlaylist.all()
query.filter('playlist =', playlist)
query.filter('live_association = ', True)
query.order('video_position')
video_playlists = query.fetch(limit)
for video_playlist in video_playlists:
videos.append(video_playlist.video)
memcache.set(key, videos, namespace=namespace)
return videos
@staticmethod
def get_cached_playlists_for_video(video, limit=5):
key = VideoPlaylist._PLAYLIST_VIDEO_KEY_FORMAT % video.key()
namespace = str(App.version) + "_" + str(
Setting.cached_library_content_date())
playlists = memcache.get(key, namespace=namespace)
if playlists is None:
playlists = []
query = VideoPlaylist.all()
query.filter('video =', video)
query.filter('live_association = ', True)
video_playlists = query.fetch(limit)
for video_playlist in video_playlists:
playlists.append(video_playlist.playlist)
memcache.set(key, playlists, namespace=namespace)
return playlists
@staticmethod
def get_query_for_playlist_title(playlist_title):
query = Playlist.all()
query.filter('title =', playlist_title)
playlist = query.get()
query = VideoPlaylist.all()
query.filter('playlist =', playlist)
query.filter(
'live_association = ', True
) #need to change this to true once I'm done with all of my hacks
query.order('video_position')
return query
@staticmethod
def get_key_dict(query):
video_playlist_key_dict = {}
for video_playlist in query.fetch(10000):
playlist_key = VideoPlaylist.playlist.get_value_for_datastore(
video_playlist)
if not video_playlist_key_dict.has_key(playlist_key):
video_playlist_key_dict[playlist_key] = {}
video_playlist_key_dict[playlist_key][
VideoPlaylist.video.get_value_for_datastore(
video_playlist)] = video_playlist
return video_playlist_key_dict
class ProductSalesData(db.Model):
product_id = db.IntegerProperty(required=True)
date = db.DateTimeProperty(verbose_name=None,
auto_now=True,
auto_now_add=True)
store = db.StringProperty(required=True)
class SummaryStatsManager(db.Model):
MEMCACHE_NAMESPACE = 'summary_stats'
#category = db.StringProperty()
summary_score = db.IntegerProperty(indexed=False)
summary_display = db.StringProperty(indexed=False)
total_runs = db.IntegerProperty(indexed=False)
@classmethod
def UpdateStats(cls, category, stats):
"""Update the summary stats in memory and the datastore.
This will only update part of a summary score row.
Args:
category: a category string like 'network'
stats: a dict of browser stats (see CategoryStatsManager.GetStats)
Returns:
The summary stats that have been updated by the given stats.
(Used by GetStats.)
"""
browsers = [b for b in stats.keys() if b != 'total_runs']
update_summary_stats = memcache.get_multi(
browsers, namespace=cls.MEMCACHE_NAMESPACE)
for browser in browsers:
ua_summary_stats = update_summary_stats.setdefault(browser, {
'results': {}})
ua_summary_stats['results'][category] = {
'score': stats[browser]['summary_score'],
'display': stats[browser]['summary_display'],
'total_runs': stats[browser]['total_runs'],
}
if category == 'acid3':
ua_summary_stats['results']['acid3']['display'] = (
stats[browser]['results']['score']['display'])
memcache.set_multi(update_summary_stats, namespace=cls.MEMCACHE_NAMESPACE)
return update_summary_stats
@classmethod
def _FindAndUpdateStats(cls, category, browsers):
test_set = all_test_sets.GetTestSet(category)
ua_stats = CategoryStatsManager.GetStats(
test_set, browsers, [t.key for t in test_set.VisibleTests()])
return cls.UpdateStats(category, ua_stats)
@classmethod
def _AddSummaryOfSummaries(cls, summary_stats):
"""Update summary_stats with row summaries."""
grand_total_runs = 0
for browser in summary_stats.keys():
results = summary_stats[browser]['results']
categories = results.keys()
score = int(sum(results[c]['score'] for c in categories)
/ len(categories))
display = '%s/100' % score
total_runs = sum(results[c]['total_runs'] for c in categories)
grand_total_runs += total_runs
summary_stats[browser].update({
'summary_score': score,
'summary_display': display,
'total_runs': total_runs,
})
summary_stats['total_runs'] = grand_total_runs
@classmethod
def GetStats(cls, browsers, categories=None):
"""Return the summary stats for a set of browsers and categories.
Gets stats out of summary memcache. If needed, re-aggregate them for the
categories. These data may come from memcache or all the way from the
datastore.
Args:
browsers: a list of browsers to use instead of version level.
categories: a list of categories like ['security', 'richtext'].
Returns:
{
browser_x: {
category_y: {
'score': score_xy,
'display': display_xy,
'total_runs': total_runs_xy,
}, ...
}, ...
}
"""
summary_stats = memcache.get_multi(
browsers, namespace=cls.MEMCACHE_NAMESPACE)
if not categories:
categories = [t.category for t in all_test_sets.GetVisibleTestSets()]
# Trim any unwanted stats and find any missing stats.
missing_stats = {}
for browser in browsers:
ua_summary_stats = summary_stats.get(browser, {'results': {}})
existing_categories = ua_summary_stats['results'].keys()
for category in existing_categories:
if category not in categories:
del ua_summary_stats['results'][category]
for category in categories:
if category not in existing_categories:
missing_stats.setdefault(category, []).append(browser)
# Load any missing stats
for category, browsers in missing_stats.items():
updated_stats = cls._FindAndUpdateStats(category, browsers)
summary_stats.update(updated_stats)
cls._AddSummaryOfSummaries(summary_stats)
return summary_stats
@classmethod
def KeyName(cls, category):
return category
class WordInfoEntity(db.Model):
word = db.StringProperty()
spamcount = db.IntegerProperty()
hamcount = db.IntegerProperty()
class MapreduceState(db.Model):
"""Holds accumulated state of mapreduce execution.
MapreduceState is stored in datastore with a key name equal to the
mapreduce ID. Only controller tasks can write to MapreduceState.
Properties:
mapreduce_spec: cached deserialized MapreduceSpec instance. read-only
active: if this MR is still running.
last_poll_time: last time controller job has polled this mapreduce.
counters_map: shard's counters map as CountersMap. Mirrors
counters_map_json.
chart_url: last computed mapreduce status chart url. This chart displays the
progress of all the shards the best way it can.
sparkline_url: last computed mapreduce status chart url in small format.
result_status: If not None, the final status of the job.
active_shards: How many shards are still processing. This starts as 0,
then set by KickOffJob handler to be the actual number of input
readers after input splitting, and is updated by Controller task
as shards finish.
start_time: When the job started.
writer_state: Json property to be used by writer to store its state.
This is filled when single output per job. Will be deprecated.
Use OutputWriter.get_filenames instead.
"""
RESULT_SUCCESS = "success"
RESULT_FAILED = "failed"
RESULT_ABORTED = "aborted"
_RESULTS = frozenset([RESULT_SUCCESS, RESULT_FAILED, RESULT_ABORTED])
# Functional properties.
# TODO(user): Replace mapreduce_spec with job_config.
mapreduce_spec = json_util.JsonProperty(MapreduceSpec, indexed=False)
active = db.BooleanProperty(default=True, indexed=False)
last_poll_time = db.DateTimeProperty(required=True)
counters_map = json_util.JsonProperty(
CountersMap, default=CountersMap(), indexed=False)
app_id = db.StringProperty(required=False, indexed=True)
writer_state = json_util.JsonProperty(dict, indexed=False)
active_shards = db.IntegerProperty(default=0, indexed=False)
failed_shards = db.IntegerProperty(default=0, indexed=False)
aborted_shards = db.IntegerProperty(default=0, indexed=False)
result_status = db.StringProperty(required=False, choices=_RESULTS)
# For UI purposes only.
chart_url = db.TextProperty(default="")
chart_width = db.IntegerProperty(default=300, indexed=False)
sparkline_url = db.TextProperty(default="")
start_time = db.DateTimeProperty(auto_now_add=True)
@classmethod
def kind(cls):
"""Returns entity kind."""
return "_AE_MR_MapreduceState"
@classmethod
def get_key_by_job_id(cls, mapreduce_id):
"""Retrieves the Key for a Job.
Args:
mapreduce_id: The job to retrieve.
Returns:
Datastore Key that can be used to fetch the MapreduceState.
"""
return db.Key.from_path(cls.kind(), str(mapreduce_id))
@classmethod
def get_by_job_id(cls, mapreduce_id):
"""Retrieves the instance of state for a Job.
Args:
mapreduce_id: The mapreduce job to retrieve.
Returns:
instance of MapreduceState for passed id.
"""
return db.get(cls.get_key_by_job_id(mapreduce_id))
def set_processed_counts(self, shards_processed):
"""Updates a chart url to display processed count for each shard.
Args:
shards_processed: list of integers with number of processed entities in
each shard
"""
chart = google_chart_api.BarChart(shards_processed)
shard_count = len(shards_processed)
if shards_processed:
# Only 16 labels on the whole chart.
stride_length = max(1, shard_count / 16)
chart.bottom.labels = []
for x in xrange(shard_count):
if (x % stride_length == 0 or
x == shard_count - 1):
chart.bottom.labels.append(x)
else:
chart.bottom.labels.append("")
chart.left.labels = ["0", str(max(shards_processed))]
chart.left.min = 0
self.chart_width = min(700, max(300, shard_count * 20))
self.chart_url = chart.display.Url(self.chart_width, 200)
def get_processed(self):
"""Number of processed entities.
Returns:
The total number of processed entities as int.
"""
return self.counters_map.get(context.COUNTER_MAPPER_CALLS)
processed = property(get_processed)
@staticmethod
def create_new(mapreduce_id=None,
gettime=datetime.datetime.now):
"""Create a new MapreduceState.
Args:
mapreduce_id: Mapreduce id as string.
gettime: Used for testing.
"""
if not mapreduce_id:
mapreduce_id = MapreduceState.new_mapreduce_id()
state = MapreduceState(key_name=mapreduce_id,
last_poll_time=gettime())
state.set_processed_counts([])
return state
@staticmethod
def new_mapreduce_id():
"""Generate new mapreduce id."""
return util._get_descending_key()
def __eq__(self, other):
if not isinstance(other, self.__class__):
return False
return self.properties() == other.properties()
class Show(db.Model):
name = db.StringProperty()
normalized_name = db.StringProperty()
alt_names = db.StringProperty()
slug = db.StringProperty()
description = db.StringProperty(indexed=False)
genres = db.StringProperty(indexed=False)
network = db.StringProperty(indexed=False)
active = db.BooleanProperty()
country = db.StringProperty(indexed=False)
runtime = db.IntegerProperty()
timezone = db.StringProperty(indexed=False)
tvrage_id = db.IntegerProperty()
added = db.DateTimeProperty()
_memkey_all_shows_ordered = "all_shows_ordered"
_memkey_shows_dict = "all_shows_dict"
re_find_the = re.compile("^The (.*)$")
@classmethod
def kind(cls):
return "series_show"
def __unicode__(self):
return self.name
@property
def idnr(self):
return self.key().id()
@property
def slug(self):
return self.normalized_name.replace(" ", "-")
def alternative_names(self):
if self.alt_names is None:
return []
return self.alt_names.split("|")
@classmethod
def get_all_ordered(cls):
shows = memcache.get(cls._memkey_all_shows_ordered)
if shows is not None:
return shows
shows = Show.all().filter("active =", True)
show_list = []
for show in shows:
if len(show.name) > 33:
show.ordered_name = cls.re_find_the.sub(
"\\1, The", show.name[:33] + "...")
else:
show.ordered_name = cls.re_find_the.sub("\\1, The", show.name)
show_list.append(show)
shows = sorted(show_list, key=lambda x: x.ordered_name.lower())
memcache.set(key=cls._memkey_all_shows_ordered, value=shows)
return shows
@classmethod
def find(cls, show_name):
if not len(show_name):
return None
norm_name = normalize(show_name)
shows = Show.get_all_ordered()
for show in shows:
if show_name == show.name or norm_name == show.normalized_name or \
any([norm_name == alt_name for alt_name in show.alternative_names()]):
return show
@classmethod
def get_all_dict(cls):
show_dict = memcache.get(cls._memkey_shows_dict)
if show_dict is not None:
return show_dict
shows = Show.get_all_ordered()
show_dict = dict([(str(show.key()), show) for show in shows])
memcache.set(key=cls._memkey_shows_dict, value=show_dict)
return show_dict
@classmethod
def clear_cache(cls):
memcache.delete(cls._memkey_all_shows_ordered)
memcache.delete(cls._memkey_shows_dict)
def add_update_task(self):
t = taskqueue.Task(url=reverse('seriesly-shows-update_show'),
params={"key": str(self.key())})
t.add(queue_name="series")
return t
def update(self, show_info=None, get_everything=False):
if show_info is None:
tvrage = TVRage()
show_info = tvrage.get_info(self.tvrage_id)
# Kill Tabatha\u2019s ... here
show_info.name = show_info.name.replace(u"\u2019", "'")
# Kill >>'Til Death<< here
if show_info.name.startswith("'"):
show_info.name = show_info.name.replace("'", "", 1)
attr_list = [
"name", "network", "genres", "active", "country", "runtime",
"timezone", "tvrage_id"
]
if self.update_attrs(show_info, attr_list):
self.put()
for season_info in show_info.seasons:
logging.debug("Update or create Season...")
Season.update_or_create(self,
season_info,
get_everything=get_everything)
def update_attrs(self, info_obj, attr_list):
changed = False
for attr in attr_list:
val = getattr(info_obj, attr)
if val != getattr(self, attr):
setattr(self, attr, val)
changed = True
return changed
def put(self):
self.normalized_name = normalize(self.name)
return super(Show, self).put()
@classmethod
def update_or_create(cls, name, show_id=None):
tvrage = TVRage()
if name is not None:
show_info = tvrage.get_info_by_name(name)
else:
show_info = tvrage.get_info(show_id)
if show_info is None:
return False
logging.debug("Show exists..?")
show = Show.all().filter("tvrage_id =", show_info.tvrage_id).get()
if show is None:
logging.debug("Creating Show...")
show = Show(name=show_info.name,
network=show_info.network,
genres=show_info.genres,
active=show_info.active,
country=show_info.country,
runtime=show_info.runtime,
timezone=show_info.timezone,
tvrage_id=show_info.tvrage_id,
added=datetime.datetime.now())
show.put()
show.update(show_info)
@property
def is_new(self):
if self.added is None:
return False
new_time = datetime.timedelta(days=7)
if datetime.datetime.now() - self.added < new_time:
return True
return False
class ShardState(db.Model):
"""Single shard execution state.
The shard state is stored in the datastore and is later aggregated by
controller task. ShardState key_name is equal to shard_id.
Shard state contains critical state to ensure the correctness of
shard execution. It is the single source of truth about a shard's
progress. For example:
1. A slice is allowed to run only if its payload matches shard state's
expectation.
2. A slice is considered running only if it has acquired the shard's lock.
3. A slice is considered done only if it has successfully committed shard
state to db.
Properties about the shard:
active: if we have this shard still running as boolean.
counters_map: shard's counters map as CountersMap. All counters yielded
within mapreduce are stored here.
mapreduce_id: unique id of the mapreduce.
shard_id: unique id of this shard as string.
shard_number: ordered number for this shard.
retries: the number of times this shard has been retried.
result_status: If not None, the final status of this shard.
update_time: The last time this shard state was updated.
shard_description: A string description of the work this shard will do.
last_work_item: A string description of the last work item processed.
writer_state: writer state for this shard. The shard's output writer
instance can save in-memory output references to this field in its
"finalize" method.
Properties about slice management:
slice_id: slice id of current executing slice. A slice's task
will not run unless its slice_id matches this. Initial
value is 0. By the end of slice execution, this number is
incremented by 1.
slice_start_time: a slice updates this to now at the beginning of
execution. If the transaction succeeds, the current task holds
a lease of slice duration + some grace period. During this time, no
other task with the same slice_id will execute. Upon slice failure,
the task should try to unset this value to allow retries to carry on
ASAP.
slice_request_id: the request id that holds/held the lease. When lease has
expired, new request needs to verify that said request has indeed
ended according to logs API. Do this only when lease has expired
because logs API is expensive. This field should always be set/unset
with slice_start_time. It is possible Logs API doesn't log a request
at all or doesn't log the end of a request. So a new request can
proceed after a long conservative timeout.
slice_retries: the number of times a slice has been retried due to
processing data when lock is held. Taskqueue/datastore errors
related to slice/shard management are not counted. This count is
only a lower bound and is used to determined when to fail a slice
completely.
acquired_once: whether the lock for this slice has been acquired at
least once. When this is True, duplicates in outputs are possible.
"""
RESULT_SUCCESS = "success"
RESULT_FAILED = "failed"
# Shard can be in aborted state when user issued abort, or controller
# issued abort because some other shard failed.
RESULT_ABORTED = "aborted"
_RESULTS = frozenset([RESULT_SUCCESS, RESULT_FAILED, RESULT_ABORTED])
# Maximum number of shard states to hold in memory at any time.
_MAX_STATES_IN_MEMORY = 10
# Functional properties.
mapreduce_id = db.StringProperty(required=True)
active = db.BooleanProperty(default=True, indexed=False)
counters_map = json_util.JsonProperty(
CountersMap, default=CountersMap(), indexed=False)
result_status = db.StringProperty(choices=_RESULTS, indexed=False)
retries = db.IntegerProperty(default=0, indexed=False)
writer_state = json_util.JsonProperty(dict, indexed=False)
slice_id = db.IntegerProperty(default=0, indexed=False)
slice_start_time = db.DateTimeProperty(indexed=False)
slice_request_id = db.ByteStringProperty(indexed=False)
slice_retries = db.IntegerProperty(default=0, indexed=False)
acquired_once = db.BooleanProperty(default=False, indexed=False)
# For UI purposes only.
update_time = db.DateTimeProperty(auto_now=True, indexed=False)
shard_description = db.TextProperty(default="")
last_work_item = db.TextProperty(default="")
def __str__(self):
kv = {"active": self.active,
"slice_id": self.slice_id,
"last_work_item": self.last_work_item,
"update_time": self.update_time}
if self.result_status:
kv["result_status"] = self.result_status
if self.retries:
kv["retries"] = self.retries
if self.slice_start_time:
kv["slice_start_time"] = self.slice_start_time
if self.slice_retries:
kv["slice_retries"] = self.slice_retries
if self.slice_request_id:
kv["slice_request_id"] = self.slice_request_id
if self.acquired_once:
kv["acquired_once"] = self.acquired_once
keys = kv.keys()
keys.sort()
result = "ShardState is {"
for k in keys:
result += k + ":" + str(kv[k]) + ","
result += "}"
return result
def reset_for_retry(self):
"""Reset self for shard retry."""
self.retries += 1
self.last_work_item = ""
self.active = True
self.result_status = None
self.counters_map = CountersMap()
self.slice_id = 0
self.slice_start_time = None
self.slice_request_id = None
self.slice_retries = 0
self.acquired_once = False
def advance_for_next_slice(self, recovery_slice=False):
"""Advance self for next slice.
Args:
recovery_slice: True if this slice is running recovery logic.
See handlers.MapperWorkerCallbackHandler._attempt_slice_recovery
for more info.
"""
self.slice_start_time = None
self.slice_request_id = None
self.slice_retries = 0
self.acquired_once = False
if recovery_slice:
self.slice_id += 2
else:
self.slice_id += 1
def set_for_failure(self):
self.active = False
self.result_status = self.RESULT_FAILED
def set_for_abort(self):
self.active = False
self.result_status = self.RESULT_ABORTED
def set_for_success(self):
self.active = False
self.result_status = self.RESULT_SUCCESS
self.slice_start_time = None
self.slice_request_id = None
self.slice_retries = 0
self.acquired_once = False
def copy_from(self, other_state):
"""Copy data from another shard state entity to self."""
for prop in self.properties().values():
setattr(self, prop.name, getattr(other_state, prop.name))
def __eq__(self, other):
if not isinstance(other, self.__class__):
return False
return self.properties() == other.properties()
def get_shard_number(self):
"""Gets the shard number from the key name."""
return int(self.key().name().split("-")[-1])
shard_number = property(get_shard_number)
def get_shard_id(self):
"""Returns the shard ID."""
return self.key().name()
shard_id = property(get_shard_id)
@classmethod
def kind(cls):
"""Returns entity kind."""
return "_AE_MR_ShardState"
@classmethod
def shard_id_from_number(cls, mapreduce_id, shard_number):
"""Get shard id by mapreduce id and shard number.
Args:
mapreduce_id: mapreduce id as string.
shard_number: shard number to compute id for as int.
Returns:
shard id as string.
"""
return "%s-%d" % (mapreduce_id, shard_number)
@classmethod
def get_key_by_shard_id(cls, shard_id):
"""Retrieves the Key for this ShardState.
Args:
shard_id: The shard ID to fetch.
Returns:
The Datatore key to use to retrieve this ShardState.
"""
return db.Key.from_path(cls.kind(), shard_id)
@classmethod
def get_by_shard_id(cls, shard_id):
"""Get shard state from datastore by shard_id.
Args:
shard_id: shard id as string.
Returns:
ShardState for given shard id or None if it's not found.
"""
return cls.get_by_key_name(shard_id)
@classmethod
def find_by_mapreduce_state(cls, mapreduce_state):
"""Find all shard states for given mapreduce.
Deprecated. Use find_all_by_mapreduce_state.
This will be removed after 1.8.9 release.
Args:
mapreduce_state: MapreduceState instance
Returns:
A list of ShardStates.
"""
return list(cls.find_all_by_mapreduce_state(mapreduce_state))
@classmethod
def find_all_by_mapreduce_state(cls, mapreduce_state):
"""Find all shard states for given mapreduce.
Args:
mapreduce_state: MapreduceState instance
Yields:
shard states sorted by shard id.
"""
keys = cls.calculate_keys_by_mapreduce_state(mapreduce_state)
i = 0
while i < len(keys):
@db.non_transactional
def no_tx_get(i):
return db.get(keys[i:i+cls._MAX_STATES_IN_MEMORY])
# We need a separate function to so that we can mix non-transactional and
# use be a generator
states = no_tx_get(i)
for s in states:
i += 1
if s is not None:
yield s
@classmethod
def calculate_keys_by_mapreduce_state(cls, mapreduce_state):
"""Calculate all shard states keys for given mapreduce.
Args:
mapreduce_state: MapreduceState instance
Returns:
A list of keys for shard states, sorted by shard id.
The corresponding shard states may not exist.
"""
if mapreduce_state is None:
return []
keys = []
for i in range(mapreduce_state.mapreduce_spec.mapper.shard_count):
shard_id = cls.shard_id_from_number(mapreduce_state.key().name(), i)
keys.append(cls.get_key_by_shard_id(shard_id))
return keys
@classmethod
def create_new(cls, mapreduce_id, shard_number):
"""Create new shard state.
Args:
mapreduce_id: unique mapreduce id as string.
shard_number: shard number for which to create shard state.
Returns:
new instance of ShardState ready to put into datastore.
"""
shard_id = cls.shard_id_from_number(mapreduce_id, shard_number)
state = cls(key_name=shard_id,
mapreduce_id=mapreduce_id)
return state
class SocialNetworks(db.Model):
code = db.IntegerProperty(required=True)
name = db.StringProperty(required=True)
class DurableJobEntity(entities.BaseEntity):
"""A class that represents a persistent database entity of durable job."""
updated_on = db.DateTimeProperty(indexed=True)
execution_time_sec = db.IntegerProperty(indexed=False)
status_code = db.IntegerProperty(indexed=False)
output = db.TextProperty(indexed=False)
sequence_num = db.IntegerProperty(indexed=False)
@classmethod
def _get_by_name(cls, name):
return DurableJobEntity.get_by_key_name(name)
@classmethod
def _update(cls, name, sequence_num, status_code, output):
"""Updates job state in a datastore."""
assert db.is_in_transaction()
job = DurableJobEntity._get_by_name(name)
if not job:
logging.error('Job was not started or was deleted: %s', name)
return
if job.sequence_num != sequence_num:
logging.warning('Request to update status code to %d ' %
status_code +
'for sequence number %d ' % sequence_num +
'but job is already on run %d' % job.sequence_num)
return
now = datetime.datetime.utcnow()
if status_code in (STATUS_CODE_STARTED, STATUS_CODE_QUEUED):
job.execution_time_sec = 0
else:
job.execution_time_sec += long(
(now - job.updated_on).total_seconds())
job.updated_on = now
job.status_code = status_code
if output:
job.output = output
job.put()
@classmethod
def _create_job(cls, name):
"""Creates new or reset a state of existing job in a datastore."""
assert db.is_in_transaction()
job = DurableJobEntity._get_by_name(name)
if not job:
job = DurableJobEntity(key_name=name)
job.updated_on = datetime.datetime.utcnow()
job.execution_time_sec = 0
job.status_code = STATUS_CODE_QUEUED
job.output = None
if not job.sequence_num:
job.sequence_num = 1
else:
job.sequence_num += 1
job.put()
return job.sequence_num
@classmethod
def _start_job(cls, name, sequence_num, output=None):
return cls._update(name, sequence_num, STATUS_CODE_STARTED, output)
@classmethod
def _complete_job(cls, name, sequence_num, output):
return cls._update(name, sequence_num, STATUS_CODE_COMPLETED, output)
@classmethod
def _fail_job(cls, name, sequence_num, output):
return cls._update(name, sequence_num, STATUS_CODE_FAILED, output)
@property
def has_finished(self):
return self.status_code in [STATUS_CODE_COMPLETED, STATUS_CODE_FAILED]
class Blogs(db.Model):
subject = db.StringProperty(required=True)
content = db.TextProperty(required=True)
created = db.DateTimeProperty(auto_now_add=True)
user_id = db.StringProperty()
likes = db.IntegerProperty(default=0)
class Counter(db.Model):
name = db.StringProperty(required=True)
count = db.IntegerProperty(required=True, default=0)
class Comment(db.Model):
blog_id = db.IntegerProperty(required=True)
comment = db.TextProperty(required=True)
commentator = db.StringProperty(required=True)
class Counter(db.Model):
"""A databased-backed counter."""
count = db.IntegerProperty(default=0)
error_possible = db.BooleanProperty(default=False)