class Post(db.Model):
title = db.StringProperty(required = True)
body = db.TextProperty(required = True)
author = db.ReferenceProperty(required = True)
created = db.DateTimeProperty(User, auto_now_add = True)
class Art(db.Model):
title = db.StringProperty(required=True)
art = db.TextProperty(required=True)
created = db.DateTimeProperty(auto_now_add=True)
coords = db.GeoPtProperty(
) #not required because would affect old art submissions
class Page(search.Searchable, db.Model):
author_name = db.StringProperty()
title = db.StringProperty()
content = db.TextProperty()
INDEX_TITLE_FROM_PROP = 'title'
class Snippet(db.Model):
user = db.ReferenceProperty(User)
text = db.TextProperty()
date = db.DateProperty()
class Blog(db.Model):
title = db.StringProperty(required=True)
body = db.TextProperty(required=True)
created = db.DateTimeProperty(auto_now_add=True)
class DurableJobEntity(entities.BaseEntity):
"""A class that represents a persistent database entity of durable job."""
updated_on = db.DateTimeProperty(indexed=True)
execution_time_sec = db.IntegerProperty(indexed=False)
status_code = db.IntegerProperty(indexed=False)
output = db.TextProperty(indexed=False)
sequence_num = db.IntegerProperty(indexed=False)
@classmethod
def _get_by_name(cls, name):
return DurableJobEntity.get_by_key_name(name)
@classmethod
def _update(cls, name, sequence_num, status_code, output):
"""Updates job state in a datastore."""
assert db.is_in_transaction()
job = DurableJobEntity._get_by_name(name)
if not job:
logging.error('Job was not started or was deleted: %s', name)
return
if job.sequence_num != sequence_num:
logging.warning('Request to update status code to %d ' %
status_code +
'for sequence number %d ' % sequence_num +
'but job is already on run %d' % job.sequence_num)
return
now = datetime.datetime.utcnow()
if status_code in (STATUS_CODE_STARTED, STATUS_CODE_QUEUED):
job.execution_time_sec = 0
else:
job.execution_time_sec += long(
(now - job.updated_on).total_seconds())
job.updated_on = now
job.status_code = status_code
if output:
job.output = output
job.put()
@classmethod
def _create_job(cls, name):
"""Creates new or reset a state of existing job in a datastore."""
assert db.is_in_transaction()
job = DurableJobEntity._get_by_name(name)
if not job:
job = DurableJobEntity(key_name=name)
job.updated_on = datetime.datetime.utcnow()
job.execution_time_sec = 0
job.status_code = STATUS_CODE_QUEUED
job.output = None
if not job.sequence_num:
job.sequence_num = 1
else:
job.sequence_num += 1
job.put()
return job.sequence_num
@classmethod
def _start_job(cls, name, sequence_num, output=None):
return cls._update(name, sequence_num, STATUS_CODE_STARTED, output)
@classmethod
def _complete_job(cls, name, sequence_num, output):
return cls._update(name, sequence_num, STATUS_CODE_COMPLETED, output)
@classmethod
def _fail_job(cls, name, sequence_num, output):
return cls._update(name, sequence_num, STATUS_CODE_FAILED, output)
@property
def has_finished(self):
return self.status_code in [STATUS_CODE_COMPLETED, STATUS_CODE_FAILED]
class _PipelineRecord(db.Model):
"""Represents a Pipeline.
Properties:
class_path: Path of the Python class to use for this pipeline.
root_pipeline: The root of the whole workflow; set to itself this pipeline
is its own root.
fanned_out: List of child _PipelineRecords that were started when this
generator pipeline moved from WAITING to RUN.
start_time: For pipelines with no start _BarrierRecord, when this pipeline
was enqueued to run immediately.
finalized_time: When this pipeline moved from WAITING or RUN to DONE.
params: Serialized parameter dictionary.
status: The current status of the pipeline.
current_attempt: The current attempt (starting at 0) to run.
max_attempts: Maximum number of attempts (starting at 0) to run.
next_retry_time: ETA of the next retry attempt.
retry_message: Why the last attempt failed; None or empty if no message.
Root pipeline properties:
is_root_pipeline: This is a root pipeline.
abort_message: Why the whole pipeline was aborted; only saved on
root pipelines.
abort_requested: If an abort signal has been requested for this root
pipeline; only saved on root pipelines
"""
WAITING = 'waiting'
RUN = 'run'
DONE = 'done'
ABORTED = 'aborted'
class_path = db.StringProperty()
root_pipeline = db.SelfReferenceProperty(
collection_name='child_pipelines_set')
fanned_out = db.ListProperty(db.Key, indexed=False)
start_time = db.DateTimeProperty(indexed=True)
finalized_time = db.DateTimeProperty(indexed=False)
# One of these two will be set, depending on the size of the params.
params_text = db.TextProperty(name='params')
params_blob = blobstore.BlobReferenceProperty(name='params_blob',
indexed=False)
status = db.StringProperty(choices=(WAITING, RUN, DONE, ABORTED),
default=WAITING)
# Retry behavior
current_attempt = db.IntegerProperty(default=0, indexed=False)
max_attempts = db.IntegerProperty(default=1, indexed=False)
next_retry_time = db.DateTimeProperty(indexed=False)
retry_message = db.TextProperty()
# Root pipeline properties
is_root_pipeline = db.BooleanProperty()
abort_message = db.TextProperty()
abort_requested = db.BooleanProperty(indexed=False)
@classmethod
def kind(cls):
return '_AE_Pipeline_Record'
@property
def params(self):
"""Returns the dictionary of parameters for this Pipeline."""
if hasattr(self, '_params_decoded'):
return self._params_decoded
if self.params_blob is not None:
value_encoded = self.params_blob.open().read()
else:
value_encoded = self.params_text
value = simplejson.loads(value_encoded, cls=util.JsonDecoder)
if isinstance(value, dict):
kwargs = value.get('kwargs')
if kwargs:
adjusted_kwargs = {}
for arg_key, arg_value in kwargs.iteritems():
# Python only allows non-unicode strings as keyword arguments.
adjusted_kwargs[str(arg_key)] = arg_value
value['kwargs'] = adjusted_kwargs
self._params_decoded = value
return self._params_decoded
def truncated_copy(self):
"""Create a lightweight copy of the pipeline with the args truncated."""
return _LowMemoryPipelineRecord(self)
class SocialGroup(db.Model):
user = db.ReferenceProperty(SocialUser)
social_account = db.ReferenceProperty(SocialAccount)
social_account_item_id = db.TextProperty()
name = db.StringProperty()
class _LogRecord(db.Model):
"""Representation of the logging information for a single web request."""
app_id = db.StringProperty()
version_id = db.StringProperty()
ip = db.StringProperty()
nickname = db.StringProperty()
request_id = db.StringProperty()
start_time = db.IntegerProperty()
end_time = db.IntegerProperty()
latency = db.IntegerProperty()
mcycles = db.IntegerProperty()
method = db.StringProperty()
resource = db.TextProperty()
status = db.IntegerProperty()
response_size = db.IntegerProperty()
http_version = db.StringProperty()
finished = db.BooleanProperty()
combined = db.TextProperty()
app_logs = db.ListProperty(db.Key)
@classmethod
def get_or_create(cls):
"""Returns the LogRecord for this request, creating it if needed."""
return cls.get_or_insert(str(_get_request_id()))
def fill_in_log(self, request, log, app_logs):
"""Fills in fields in a given RequestLog from a LogReadRequest's fields.
Application-level logs are stored in the Datastore as _LogLines, so this
method also grabs those items, resolves them, and stores them in the given
RequestLog.
Args:
request: A LogReadRequest, containing the filters that the user has
specified to use for their request.
log: A RequestLog whose fields will be overriden with those from request.
app_logs: The application-level logs associated with the given log.
"""
log.set_app_id(self.app_id)
log.set_version_id(self.version_id)
log.set_ip(self.ip)
log.set_nickname(self.nickname)
log.set_request_id(self.request_id)
log.set_start_time(self.start_time)
log.set_end_time(self.end_time)
log.set_latency(self.latency)
log.set_mcycles(self.mcycles)
log.set_method(self.method)
log.set_resource(self.resource)
log.set_status(self.status)
log.set_response_size(self.response_size)
log.set_http_version(self.http_version)
log.set_finished(self.finished)
log.set_combined(self.combined)
if request.include_app_logs():
for app_log in app_logs:
log_line = log.add_line()
log_line.set_time(app_log.time)
log_line.set_level(app_log.level)
log_line.set_log_message(app_log.message)
class Quotes(db.Model):
quote=db.TextProperty(required=True)
author=db.StringProperty(required=True)
class MyUser(db.Model):
email = db.EmailProperty()
display_name = db.TextProperty()
past_view_count = db.IntegerProperty(
default=0) # just for demo purposes ...
class Posts(db.Model):
subject = db.StringProperty(required=True)
content = db.TextProperty(required=True)
created = db.DateTimeProperty(auto_now_add=True)
class settingsbd(db.Model): keyS = db.StringProperty() valueS = db.TextProperty()
class Art(db.Model):
title = db.StringProperty(required=True)
art = db.TextProperty(required=True)
created = db.DateTimeProperty(auto_now_add=True)
coords = db.GeoPtProperty()
class ShardState(db.Model):
"""Single shard execution state.
The shard state is stored in the datastore and is later aggregated by
controller task. ShardState key_name is equal to shard_id.
Shard state contains critical state to ensure the correctness of
shard execution. It is the single source of truth about a shard's
progress. For example:
1. A slice is allowed to run only if its payload matches shard state's
expectation.
2. A slice is considered running only if it has acquired the shard's lock.
3. A slice is considered done only if it has successfully committed shard
state to db.
Properties about the shard:
active: if we have this shard still running as boolean.
counters_map: shard's counters map as CountersMap. All counters yielded
within mapreduce are stored here.
mapreduce_id: unique id of the mapreduce.
shard_id: unique id of this shard as string.
shard_number: ordered number for this shard.
retries: the number of times this shard has been retried.
result_status: If not None, the final status of this shard.
update_time: The last time this shard state was updated.
shard_description: A string description of the work this shard will do.
last_work_item: A string description of the last work item processed.
writer_state: writer state for this shard. The shard's output writer
instance can save in-memory output references to this field in its
"finalize" method.
Properties about slice management:
slice_id: slice id of current executing slice. A slice's task
will not run unless its slice_id matches this. Initial
value is 0. By the end of slice execution, this number is
incremented by 1.
slice_start_time: a slice updates this to now at the beginning of
execution. If the transaction succeeds, the current task holds
a lease of slice duration + some grace period. During this time, no
other task with the same slice_id will execute. Upon slice failure,
the task should try to unset this value to allow retries to carry on
ASAP.
slice_request_id: the request id that holds/held the lease. When lease has
expired, new request needs to verify that said request has indeed
ended according to logs API. Do this only when lease has expired
because logs API is expensive. This field should always be set/unset
with slice_start_time. It is possible Logs API doesn't log a request
at all or doesn't log the end of a request. So a new request can
proceed after a long conservative timeout.
slice_retries: the number of times a slice has been retried due to
processing data when lock is held. Taskqueue/datastore errors
related to slice/shard management are not counted. This count is
only a lower bound and is used to determined when to fail a slice
completely.
acquired_once: whether the lock for this slice has been acquired at
least once. When this is True, duplicates in outputs are possible.
"""
RESULT_SUCCESS = "success"
RESULT_FAILED = "failed"
# Shard can be in aborted state when user issued abort, or controller
# issued abort because some other shard failed.
RESULT_ABORTED = "aborted"
_RESULTS = frozenset([RESULT_SUCCESS, RESULT_FAILED, RESULT_ABORTED])
# Maximum number of shard states to hold in memory at any time.
_MAX_STATES_IN_MEMORY = 10
# Functional properties.
mapreduce_id = db.StringProperty(required=True)
active = db.BooleanProperty(default=True, indexed=False)
counters_map = json_util.JsonProperty(
CountersMap, default=CountersMap(), indexed=False)
result_status = db.StringProperty(choices=_RESULTS, indexed=False)
retries = db.IntegerProperty(default=0, indexed=False)
writer_state = json_util.JsonProperty(dict, indexed=False)
slice_id = db.IntegerProperty(default=0, indexed=False)
slice_start_time = db.DateTimeProperty(indexed=False)
slice_request_id = db.ByteStringProperty(indexed=False)
slice_retries = db.IntegerProperty(default=0, indexed=False)
acquired_once = db.BooleanProperty(default=False, indexed=False)
# For UI purposes only.
update_time = db.DateTimeProperty(auto_now=True, indexed=False)
shard_description = db.TextProperty(default="")
last_work_item = db.TextProperty(default="")
def __str__(self):
kv = {"active": self.active,
"slice_id": self.slice_id,
"last_work_item": self.last_work_item,
"update_time": self.update_time}
if self.result_status:
kv["result_status"] = self.result_status
if self.retries:
kv["retries"] = self.retries
if self.slice_start_time:
kv["slice_start_time"] = self.slice_start_time
if self.slice_retries:
kv["slice_retries"] = self.slice_retries
if self.slice_request_id:
kv["slice_request_id"] = self.slice_request_id
if self.acquired_once:
kv["acquired_once"] = self.acquired_once
keys = kv.keys()
keys.sort()
result = "ShardState is {"
for k in keys:
result += k + ":" + str(kv[k]) + ","
result += "}"
return result
def reset_for_retry(self):
"""Reset self for shard retry."""
self.retries += 1
self.last_work_item = ""
self.active = True
self.result_status = None
self.counters_map = CountersMap()
self.slice_id = 0
self.slice_start_time = None
self.slice_request_id = None
self.slice_retries = 0
self.acquired_once = False
def advance_for_next_slice(self, recovery_slice=False):
"""Advance self for next slice.
Args:
recovery_slice: True if this slice is running recovery logic.
See handlers.MapperWorkerCallbackHandler._attempt_slice_recovery
for more info.
"""
self.slice_start_time = None
self.slice_request_id = None
self.slice_retries = 0
self.acquired_once = False
if recovery_slice:
self.slice_id += 2
else:
self.slice_id += 1
def set_for_failure(self):
self.active = False
self.result_status = self.RESULT_FAILED
def set_for_abort(self):
self.active = False
self.result_status = self.RESULT_ABORTED
def set_for_success(self):
self.active = False
self.result_status = self.RESULT_SUCCESS
self.slice_start_time = None
self.slice_request_id = None
self.slice_retries = 0
self.acquired_once = False
def copy_from(self, other_state):
"""Copy data from another shard state entity to self."""
for prop in self.properties().values():
setattr(self, prop.name, getattr(other_state, prop.name))
def __eq__(self, other):
if not isinstance(other, self.__class__):
return False
return self.properties() == other.properties()
def get_shard_number(self):
"""Gets the shard number from the key name."""
return int(self.key().name().split("-")[-1])
shard_number = property(get_shard_number)
def get_shard_id(self):
"""Returns the shard ID."""
return self.key().name()
shard_id = property(get_shard_id)
@classmethod
def kind(cls):
"""Returns entity kind."""
return "_AE_MR_ShardState"
@classmethod
def shard_id_from_number(cls, mapreduce_id, shard_number):
"""Get shard id by mapreduce id and shard number.
Args:
mapreduce_id: mapreduce id as string.
shard_number: shard number to compute id for as int.
Returns:
shard id as string.
"""
return "%s-%d" % (mapreduce_id, shard_number)
@classmethod
def get_key_by_shard_id(cls, shard_id):
"""Retrieves the Key for this ShardState.
Args:
shard_id: The shard ID to fetch.
Returns:
The Datatore key to use to retrieve this ShardState.
"""
return db.Key.from_path(cls.kind(), shard_id)
@classmethod
def get_by_shard_id(cls, shard_id):
"""Get shard state from datastore by shard_id.
Args:
shard_id: shard id as string.
Returns:
ShardState for given shard id or None if it's not found.
"""
return cls.get_by_key_name(shard_id)
@classmethod
def find_by_mapreduce_state(cls, mapreduce_state):
"""Find all shard states for given mapreduce.
Deprecated. Use find_all_by_mapreduce_state.
This will be removed after 1.8.9 release.
Args:
mapreduce_state: MapreduceState instance
Returns:
A list of ShardStates.
"""
return list(cls.find_all_by_mapreduce_state(mapreduce_state))
@classmethod
def find_all_by_mapreduce_state(cls, mapreduce_state):
"""Find all shard states for given mapreduce.
Args:
mapreduce_state: MapreduceState instance
Yields:
shard states sorted by shard id.
"""
keys = cls.calculate_keys_by_mapreduce_state(mapreduce_state)
i = 0
while i < len(keys):
@db.non_transactional
def no_tx_get(i):
return db.get(keys[i:i+cls._MAX_STATES_IN_MEMORY])
# We need a separate function to so that we can mix non-transactional and
# use be a generator
states = no_tx_get(i)
for s in states:
i += 1
if s is not None:
yield s
@classmethod
def calculate_keys_by_mapreduce_state(cls, mapreduce_state):
"""Calculate all shard states keys for given mapreduce.
Args:
mapreduce_state: MapreduceState instance
Returns:
A list of keys for shard states, sorted by shard id.
The corresponding shard states may not exist.
"""
if mapreduce_state is None:
return []
keys = []
for i in range(mapreduce_state.mapreduce_spec.mapper.shard_count):
shard_id = cls.shard_id_from_number(mapreduce_state.key().name(), i)
keys.append(cls.get_key_by_shard_id(shard_id))
return keys
@classmethod
def create_new(cls, mapreduce_id, shard_number):
"""Create new shard state.
Args:
mapreduce_id: unique mapreduce id as string.
shard_number: shard number for which to create shard state.
Returns:
new instance of ShardState ready to put into datastore.
"""
shard_id = cls.shard_id_from_number(mapreduce_id, shard_number)
state = cls(key_name=shard_id,
mapreduce_id=mapreduce_id)
return state
class _LogLine(db.Model): """Representation of an application-level log line.""" time = db.IntegerProperty() level = db.IntegerProperty() message = db.TextProperty()
class FeedbackSettings(db.Model):
From = db.StringProperty()
ValidRecipients = db.StringListProperty()
StyleSheet = db.TextProperty()
ReCaptchaPublic = db.StringProperty()
ReCaptchaPrivate = db.StringProperty()
class Election(db.Model):
"""
An election that users may vote for.
"""
name = db.StringProperty(required=True)
start = db.DateTimeProperty(required=True) # Time when voting begins
end = db.DateTimeProperty(required=True) # Time when voting ends
organization = db.ReferenceProperty(Organization,
collection_name='elections')
result_computed = db.BooleanProperty(required=True, default=False)
result_delay = db.IntegerProperty(
required=True, default=0) # Results delay to public in seconds
universal = db.BooleanProperty(required=True, default=False)
hidden = db.BooleanProperty(required=True, default=False)
task_count = db.IntegerProperty(
required=True, # Counter used to identify tasks
default=0)
voter_count = db.IntegerProperty(required=True, default=0)
voted_count = db.IntegerProperty(required=True, default=0)
description = db.TextProperty(required=False, default="")
def to_json(self, parseable_date=False):
# calculate the publication date by adding result delay to end
pub = self.end + timedelta(seconds=self.result_delay)
# parseable_date means it can be parsed in JS with Date.parse()
times = {
'start': self.start.strftime('%a, %B %d, %Y, %I:%M %p') + ' UTC',
'end': self.end.strftime('%a, %B %d, %Y, %I:%M %p') + ' UTC',
'pub': pub.strftime('%a, %B %d, %Y, %I:%M %p') + ' UTC'
} if parseable_date else {
'start': str(self.start),
'end': str(self.end),
'pub': str(pub)
}
now = datetime.now()
status = 'Not started'
if now > self.start:
status = 'Voting in progress'
if now > self.end:
status = 'Voting has ended'
if self.result_computed:
status = 'Result computed'
return {
'id': str(self.key()),
'name': self.name,
'organization': self.organization.name,
'times': times,
'result_computed': self.result_computed,
'result_delay': self.result_delay,
'universal': self.universal,
'hidden': self.hidden,
'voter_count': self.voter_count,
'voted_count': self.voted_count,
'description': self.description,
'status': status,
}
@property
def election_positions(self):
return [
ep for ep in ElectionPosition.gql(
"WHERE election=:1 ORDER BY datetime_created", self)
]
class Post(db.Model):
title = db.StringProperty()
author = db.UserProperty()
content = db.TextProperty()
format = db.StringProperty()
date = db.DateTimeProperty()
class Membership(db.Model):
hash = db.StringProperty()
first_name = db.StringProperty(required=True)
last_name = db.StringProperty(required=True)
email = db.StringProperty(required=True)
# The hash of the user's password.
# TODO(danielp): Make this required after we finish migrating away from domain
# accounts.
password_hash = db.StringProperty()
twitter = db.StringProperty(required=False)
plan = db.StringProperty(required=False)
status = db.StringProperty() # None, active, suspended
referuserid = db.StringProperty()
referrer = db.StringProperty()
rfid_tag = db.StringProperty()
extra_599main = db.StringProperty()
extra_dnd = db.BooleanProperty(default=False)
auto_signin = db.StringProperty()
unsubscribe_reason = db.TextProperty()
hardship_comment = db.TextProperty()
spreedly_token = db.StringProperty()
parking_pass = db.StringProperty()
created = db.DateTimeProperty(auto_now_add=True)
updated = db.DateTimeProperty()
# How many times the user has signed in this month.
signins = db.IntegerProperty(default=0)
# When the last time they signed in was.
last_signin = db.DateTimeProperty()
# The following are legacy parameters.
# TODO(danielp): Remove these after we complete the migration away from
# domain accounts.
# Whether we've created a google apps user yet.
domain_user = db.BooleanProperty(default=False)
# The user's domain username.
username = db.StringProperty()
# Temporarily stores the user's domain password.
password = db.StringProperty(default=None)
""" Override of the default put method which allows us to skip changing the
updated property for testing purposes.
skip_time_update: Whether or not to set updated to the current date and time.
"""
def put(self, *args, **kwargs):
if not kwargs.pop("skip_time_update", False):
self.updated = datetime.datetime.now()
super(Membership, self).put(*args, **kwargs)
def icon(self):
return str("http://www.gravatar.com/avatar/" + hashlib.md5(self.email.lower()).hexdigest())
def full_name(self):
return '%s %s' % (self.first_name, self.last_name)
def spreedly_url(self):
config = Config()
return str("https://subs.pinpayments.com/%s/subscriber_accounts/%s" % \
(config.SPREEDLY_ACCOUNT, self.spreedly_token))
def spreedly_admin_url(self):
config = Config()
return str("https://subs.pinpayments.com/%s/subscribers/%s" % \
(config.SPREEDLY_ACCOUNT, self.key().id()))
def subscribe_url(self, plan=None):
config = Config()
if not plan:
plan = self.plan
url = "https://subs.pinpayments.com/%s/subscribers/%i/%s/subscribe/%s" % \
(config.SPREEDLY_ACCOUNT, self.key().id(),
self.spreedly_token, plans.Plan.get_by_name(plan).plan_id)
return str(url)
""" URL we use to subscribe a person for the first time.
host: The first part of the return URL, e.g. signup.hackerdojo.com.
plan: Optionally specifies a different plan to use. """
def new_subscribe_url(self, host, plan=None):
config = Config()
if not plan:
plan = self.plan
query_str = urllib.urlencode({"first_name": self.first_name,
"last_name": self.last_name,
"email": self.email,
"return_url": "http://%s/success/%s" % \
(host, self.hash)})
url = "https://subs.pinpayments.com/%s/subscribers/%i/subscribe/%s/%s?%s" % \
(config.SPREEDLY_ACCOUNT, self.key().id(),
plans.Plan.get_by_name(plan).plan_id, self.username, query_str)
return str(url)
def force_full_subscribe_url(self):
config = Config()
url = "https://subs.pinpayments.com/%s/subscribers/%i/%s/subscribe/%s" % \
(config.SPREEDLY_ACCOUNT, self.key().id(),
self.spreedly_token, plans.newfull.plan_id)
return str(url)
def unsubscribe_url(self):
return "http://signup.hackerdojo.com/unsubscribe/%i" % (self.key().id())
""" Returns this user's unique ID, which can be an integer or string. """
def get_id(self):
return self.key().id()
""" Sets the user's password.
password: The password which will be hashed and stored. """
def set_password(self, password):
logging.debug("Setting password for user %s." % (self.email))
self.password_hash = security.generate_password_hash(password, length=12)
""" Creates a new authorization token for a given user ID.
user_id: User unique ID.
Returns: A string with the authorization token. """
@classmethod
def create_auth_token(cls, user_id):
token = UserToken(user_id, "auth")
token.save()
return token.token
""" Deletes a given authorization token.
user_id: User unique ID.
token: A string with the authorization token. """
@classmethod
def delete_auth_token(cls, user_id, token):
token = UserToken.verify(user_id, "auth", token)
if not token:
logging.warning("Delete: Ignoring bad token for %d." % (user_id))
return
token.delete()
""" Returns a Membership object based on a user ID and token.
user_id: The unique ID of the requesting user.
token: The token string to be verified.
Returns: A tuple (Membership, timestamp), with a Membership object and
the token timestamp, or (None, None) if both were not found. """
@classmethod
def get_by_auth_token(cls, user_id, token):
# First, check that the token is valid.
token = UserToken.verify(user_id, "auth", token)
if not token:
logging.warning("Bad token, not getting user %d." % (user_id))
return (None, None)
user = cls.get_by_id(user_id)
timestamp = token.timestamp
return (user, timestamp)
""" Gets the user with the specified login credentials.
email: The email of the user.
password: The password of the user.
Returns: Membership object if found. """
@classmethod
def get_by_auth_password(cls, email, password):
user = cls.get_by_email(email)
if not user:
raise auth.InvalidAuthIdError("No user with email '%s'." % (email))
if not security.check_password_hash(password, user.password_hash):
raise auth.InvalidPasswordError("Bad password for user '%s'." % (email))
return user
""" Gets the user with the specified email.
email: Either the normal email, or the hackerdojo.com email of the user.
Returns: The membership object corresponding to the user, or None if no user
was found. """
@classmethod
def get_by_email(cls, email):
# TODO(danielp): Remove code for dealing with hackerdojo.com emails after
# we've finished migrating away from domain accounts.
if "@hackerdojo.com" in email:
username = email.split("@")[0]
return cls.get_by_username(username)
return cls.all().filter('email =', email).get()
@classmethod
def get_by_hash(cls, hash):
return cls.all().filter('hash =', hash).get()
# This is a legacy method:
# TODO(danielp): Remove this after we migrate away from domain accounts.
@classmethod
def get_by_username(cls, username):
return cls.all().filter('username ='******'s email. This will be used as a unique ID.
password: The user's raw password. Will be hashed before saving, obviously.
other_properties: Keyword arguments specifying properties that will be
forwarded to the Membership constructor. All the other required properties
should be in here.
Returns: The created Membership entity. """
@classmethod
def create_user(cls, email, password, **other_properties):
logging.info("Creating user with email '%s', other properties: %s." % \
(email, other_properties))
password_hash = security.generate_password_hash(password, length=12)
member = cls(email=email, password_hash=password_hash, **other_properties)
member.put()
return member
class BlogInput(db.Model):
subject = db.StringProperty(required=True)
content = db.TextProperty(required=True)
created = db.DateTimeProperty(auto_now_add=True)
date = db.DateProperty(auto_now_add=True)
class Issue(db.Model):
"""Represents a single issue which is being voted on"""
status = db.StringProperty(required=True,
default='active',
choices=set(['active', 'done', 'canceled']))
is_public = db.BooleanProperty(
default=False) #is issue listed on site or just by sharing url?
creator = db.UserProperty(auto_current_user=True)
title = db.StringProperty(required=True)
description = db.TextProperty()
duration = db.IntegerProperty()
creation_date = db.DateTimeProperty(auto_now_add=True)
start_time = db.DateTimeProperty() #time when first vote is cast
end_time = db.DateTimeProperty() #time when vote will end
#Implicit Properties:
#choices = Implicitly created list of choice objects
#votes - implicitly created list of vote objects
def add_choice(self, choice_name):
new_choice = Choice(name=choice_name, issue=self)
new_choice.put()
def remove_choice(self, choice):
choice.delete()
def vote_count(self):
return self.votes.count(999999)
def vote_for_member(self, member=None):
if not member:
member = users.get_current_user()
logging.info('member:%s voted:%s' %
(member.nickname(), self.votes.filter('member =',
member).fetch(20)))
return self.votes.filter('member =', member).get()
def register_vote(self, choice, member=None):
if not member:
member = users.get_current_user()
member_vote = self.vote_for_member(member)
was_changed = False
if (member_vote):
member_vote.choice = choice
was_changed = True
else:
member_vote = Vote(member=member, choice=choice, issue=self)
member_vote.put()
if (not self.start_time):
self.start_time = datetime.now()
self.end_time = self.start_time + timedelta(hours=self.duration)
self.put()
return was_changed
def extend_duration(self, hours):
self.duration += hours
if self.start_time:
self.end_time = self.start_time + timedelta(hours=self.duration)
self.put()
def days_left(self):
delta = self.end_time - datetime.now()
return delta.days
def hours_left(self):
delta = self.end_time - datetime.now()
#days = delta.days
#hours = days*24 + delta.seconds/3600
hours = delta.seconds / 3600
return hours
def is_active(self):
return self.status in ('active')
def has_results(self):
return self.status in ('done')
def member_is_creator(self, member=None):
if not member:
member = users.get_current_user()
return member == self.creator
def winning_choices(
self): #returns list of keys of winning choices (may be a tie)
result = []
high_vote = 0
for choice in self.choices:
cnt = choice.vote_count()
if cnt == high_vote:
result.append(choice.key())
elif cnt > high_vote:
result = [choice.key()]
high_vote = cnt
return result
def update_status(self):
if self.is_active:
if self.end_time:
if self.end_time <= datetime.now():
logging.info('status changed for issue: %s' % (self.title))
self.status = 'done'
self.put()
@classmethod
def issues_created_by(cls, member=None, limit=20):
if not member:
member = users.get_current_user()
return cls.all().filter('creator =',
member).order('-creation_date').fetch(limit)
@classmethod
def issues_voted_on(cls, member=None, limit=20):
if not member:
member = users.get_current_user()
if not member: #if logged out
return []
member_votes = Vote.all().filter(
'member =', member).order('-update_time').fetch(limit)
##logging.info('member_votes:%s' % (member_votes))
##logging.info('output:%s' % ([vote.issue for vote in member_votes]))
return [vote.issue for vote in member_votes]
@classmethod
def recent_results(
cls,
member=None,
limit=20
): #*** Need to fix, limit will be incorrect here because of filtering
if not member:
member = users.get_current_user()
if not member: #if logged out
return []
recent = cls.all().filter('status =',
'done').order('-end_time').fetch(limit)
return [issue for issue in recent
if issue.vote_for_member()] #***this is probably slow
class BlogEntries(db.Model):
title = db.StringProperty(required = True)
content = db.TextProperty(required = True)
created = db.DateTimeProperty(auto_now_add = True)
class Video(Searchable, db.Model):
youtube_id = db.StringProperty()
url = db.StringProperty()
title = db.StringProperty()
description = db.TextProperty()
playlists = db.StringListProperty()
keywords = db.StringProperty()
duration = db.IntegerProperty(default=0)
# Human readable, unique id that can be used in URLS.
readable_id = db.StringProperty()
# YouTube view count from last sync.
views = db.IntegerProperty(default=0)
# Date first added via KA library sync with YouTube.
# This property hasn't always existsed, so for many old videos
# this date may be much later than the actual YouTube upload date.
date_added = db.DateTimeProperty(auto_now_add=True)
INDEX_ONLY = ['title', 'keywords', 'description']
INDEX_TITLE_FROM_PROP = 'title'
INDEX_USES_MULTI_ENTITIES = False
@staticmethod
def get_for_readable_id(readable_id):
video = None
query = Video.all()
query.filter('readable_id =', readable_id)
# The following should just be:
# video = query.get()
# but the database currently contains multiple Video objects for a particular
# video. Some are old. Some are due to a YouTube sync where the youtube urls
# changed and our code was producing youtube_ids that ended with '_player'.
# This hack gets the most recent valid Video object.
key_id = 0
for v in query:
if v.key().id() > key_id and not v.youtube_id.endswith('_player'):
video = v
key_id = v.key().id()
# End of hack
return video
def first_playlist(self):
query = VideoPlaylist.all()
query.filter('video =', self)
query.filter('live_association =', True)
video_playlist = query.get()
if video_playlist:
return video_playlist.playlist
return None
def current_user_points(self):
user_video = UserVideo.get_for_video_and_user(self,
util.get_current_user())
if user_video:
return points.VideoPointCalculator(user_video)
else:
return 0
@staticmethod
def get_dict(query, fxn_key):
video_dict = {}
for video in query.fetch(10000):
video_dict[fxn_key(video)] = video
return video_dict
class Page(db.Model):
uri = db.StringProperty()
#html metadata
title = db.StringProperty()
description = db.StringProperty()
keywords = db.StringProperty()
#html content
date = db.DateTimeProperty()
date_last_edited = db.DateTimeProperty()
headline = db.StringProperty()
img = db.StringProperty()
img_square_css = db.StringProperty()
list_display = db.StringProperty()
# see https://cdnjs.com/libraries/highlight.js/ for list
syntaxes = db.StringProperty()
def get_syntax_list(self):
return get_syntax_list([self])
snippet = db.TextProperty()
snippet_is_markdown = db.BooleanProperty(default=True)
content = db.TextProperty()
content_is_markdown = db.BooleanProperty(default=True)
def display_date(self):
if self.date:
if self.date_last_edited:
return '{} (updated {})'.format(self.date,
self.date_last_edited)
else:
return self.date
elif self.date_last_edited:
return 'updated {}'.format(self.date_last_edited)
else:
return None
def display_date_short(self):
if self.date_last_edited:
return 'updated ' + self.date_last_edited.split('-')[0]
elif self.date:
return self.date.split('-')[0]
else:
return None
def get_snippet(self):
if self.snippet:
return self.snippet
elif not self.content:
return None
else:
break_text = '<br id="break"/>'
index_of_break = self.content.find(break_text)
if index_of_break < 0:
return None
else:
return self.content[0:index_of_break]
#organization
list_id = db.StringProperty()
precedence = db.FloatProperty(default=1.0)
is_public = db.BooleanProperty(default=False)
def load(self, **entries):
self.__dict__.update(entries)
class Exercise(db.Model):
name = db.StringProperty()
short_display_name = db.StringProperty(default="")
prerequisites = db.StringListProperty()
covers = db.StringListProperty()
v_position = db.IntegerProperty()
h_position = db.IntegerProperty()
seconds_per_fast_problem = db.FloatProperty(
default=consts.MIN_SECONDS_PER_FAST_PROBLEM
) # Seconds expected to finish a problem 'quickly' for badge calculation
# True if this exercise is live and visible to all users.
# Non-live exercises are only visible to admins.
live = db.BooleanProperty(default=False)
# True if this exercise is a quasi-exercise generated by
# combining the content of other exercises
summative = db.BooleanProperty(default=False)
# Teachers contribute raw html with embedded CSS and JS
# and we sanitize it with Caja before displaying it to
# students.
author = db.UserProperty()
raw_html = db.TextProperty()
last_modified = db.DateTimeProperty()
safe_html = db.TextProperty()
safe_js = db.TextProperty()
last_sanitized = db.DateTimeProperty(default=datetime.datetime.min)
sanitizer_used = db.StringProperty()
@staticmethod
def get_by_name(name):
dict_exercises = Exercise.__get_dict_use_cache_unsafe__()
if dict_exercises.has_key(name):
if dict_exercises[name].is_visible_to_current_user():
return dict_exercises[name]
return None
@staticmethod
def to_display_name(name):
if name:
return name.replace('_', ' ').capitalize()
return ""
def display_name(self):
return Exercise.to_display_name(self.name)
@staticmethod
def to_short_name(name):
exercise = Exercise.get_by_name(name)
if exercise:
return exercise.short_name()
return ""
def short_name(self):
if self.short_display_name:
return self.short_display_name[:11]
return self.display_name()[:11]
def is_visible_to_current_user(self):
return self.live or users.is_current_user_admin()
def required_streak(self):
if self.summative:
return consts.REQUIRED_STREAK * len(self.prerequisites)
else:
return consts.REQUIRED_STREAK
def struggling_threshold(self):
return 3 * self.required_streak()
def summative_children(self):
if not self.summative:
return []
query = db.Query(Exercise)
query.filter("name IN ", self.prerequisites)
return query
def non_summative_exercise(self, problem_number):
if not self.summative:
return self
if len(self.prerequisites) <= 0:
raise Exception(
"Summative exercise '%s' does not include any other exercises"
% self.name)
# For now we just cycle through all of the included exercises in a summative exercise
index = int(problem_number) % len(self.prerequisites)
exid = self.prerequisites[index]
query = Exercise.all()
query.filter('name =', exid)
exercise = query.get()
if not exercise:
raise Exception("Unable to find included exercise")
if exercise.summative:
return exercise.non_summative_exercise(problem_number)
else:
return exercise
def related_videos(self):
exercise_videos = None
query = ExerciseVideo.all()
query.filter('exercise =', self.key())
return query
@layer_cache.cache_with_key_fxn(
lambda self: "related_videos_%s" % self.key(),
layer=layer_cache.SINGLE_LAYER_MEMCACHE_ONLY)
def related_videos_fetch(self):
exercise_videos = self.related_videos().fetch(10)
for exercise_video in exercise_videos:
exercise_video.video # Pre-cache video entity
return exercise_videos
_CURRENT_SANITIZER = "http://caja.appspot.com/"
def ensure_sanitized(self):
if self.last_sanitized >= self.last_modified and self.sanitizer_used == Exercise._CURRENT_SANITIZER:
return
cajoled = cajole.cajole(self.raw_html)
if 'error' in cajoled:
raise Exception(cajoled['html'])
self.safe_html = db.Text(cajoled['html'])
self.safe_js = db.Text(cajoled['js'])
self.last_sanitized = datetime.datetime.now()
self.sanitizer = Exercise._CURRENT_SANITIZER
self.put()
@classmethod
def all(cls, live_only=False):
query = super(Exercise, cls).all()
if live_only or not users.is_current_user_admin():
query.filter("live =", True)
return query
@classmethod
def all_unsafe(cls):
return super(Exercise, cls).all()
@staticmethod
def get_all_use_cache():
if users.is_current_user_admin():
return Exercise.__get_all_use_cache_unsafe__()
else:
return Exercise.__get_all_use_cache_safe__()
@staticmethod
@layer_cache.cache_with_key_fxn(
lambda *args, **kwargs: "all_exercises_unsafe_%s" % Setting.
cached_exercises_date())
def __get_all_use_cache_unsafe__():
query = Exercise.all_unsafe().order('h_position')
return query.fetch(200)
@staticmethod
@layer_cache.cache_with_key_fxn(
lambda *args, **kwargs: "all_exercises_safe_%s" % Setting.
cached_exercises_date())
def __get_all_use_cache_safe__():
query = Exercise.all(live_only=True).order('h_position')
return query.fetch(200)
@staticmethod
@layer_cache.cache_with_key_fxn(
lambda *args, **kwargs: "all_exercises_dict_unsafe_%s" % Setting.
cached_exercises_date())
def __get_dict_use_cache_unsafe__():
exercises = Exercise.__get_all_use_cache_unsafe__()
dict_exercises = {}
for exercise in exercises:
dict_exercises[exercise.name] = exercise
return dict_exercises
_EXERCISES_COUNT_KEY = "Exercise.count()"
@staticmethod
def get_count():
count = memcache.get(Exercise._EXERCISES_COUNT_KEY,
namespace=App.version)
if count is None:
count = Exercise.all().count()
memcache.set(Exercise._EXERCISES_COUNT_KEY,
count,
namespace=App.version)
return count
def put(self):
Setting.cached_exercises_date(str(datetime.datetime.now()))
memcache.delete(Exercise._EXERCISES_COUNT_KEY, namespace=App.version)
db.Model.put(self)
class BQueryModel(db.Model):
author = db.UserProperty()
querystr = db.TextProperty()
qdesc = db.TextProperty()
date = db.DateTimeProperty(auto_now_add=True)
class MapreduceState(db.Model):
"""Holds accumulated state of mapreduce execution.
MapreduceState is stored in datastore with a key name equal to the
mapreduce ID. Only controller tasks can write to MapreduceState.
Properties:
mapreduce_spec: cached deserialized MapreduceSpec instance. read-only
active: if this MR is still running.
last_poll_time: last time controller job has polled this mapreduce.
counters_map: shard's counters map as CountersMap. Mirrors
counters_map_json.
chart_url: last computed mapreduce status chart url. This chart displays the
progress of all the shards the best way it can.
sparkline_url: last computed mapreduce status chart url in small format.
result_status: If not None, the final status of the job.
active_shards: How many shards are still processing. This starts as 0,
then set by KickOffJob handler to be the actual number of input
readers after input splitting, and is updated by Controller task
as shards finish.
start_time: When the job started.
writer_state: Json property to be used by writer to store its state.
This is filled when single output per job. Will be deprecated.
Use OutputWriter.get_filenames instead.
"""
RESULT_SUCCESS = "success"
RESULT_FAILED = "failed"
RESULT_ABORTED = "aborted"
_RESULTS = frozenset([RESULT_SUCCESS, RESULT_FAILED, RESULT_ABORTED])
# Functional properties.
# TODO(user): Replace mapreduce_spec with job_config.
mapreduce_spec = json_util.JsonProperty(MapreduceSpec, indexed=False)
active = db.BooleanProperty(default=True, indexed=False)
last_poll_time = db.DateTimeProperty(required=True)
counters_map = json_util.JsonProperty(
CountersMap, default=CountersMap(), indexed=False)
app_id = db.StringProperty(required=False, indexed=True)
writer_state = json_util.JsonProperty(dict, indexed=False)
active_shards = db.IntegerProperty(default=0, indexed=False)
failed_shards = db.IntegerProperty(default=0, indexed=False)
aborted_shards = db.IntegerProperty(default=0, indexed=False)
result_status = db.StringProperty(required=False, choices=_RESULTS)
# For UI purposes only.
chart_url = db.TextProperty(default="")
chart_width = db.IntegerProperty(default=300, indexed=False)
sparkline_url = db.TextProperty(default="")
start_time = db.DateTimeProperty(auto_now_add=True)
@classmethod
def kind(cls):
"""Returns entity kind."""
return "_AE_MR_MapreduceState"
@classmethod
def get_key_by_job_id(cls, mapreduce_id):
"""Retrieves the Key for a Job.
Args:
mapreduce_id: The job to retrieve.
Returns:
Datastore Key that can be used to fetch the MapreduceState.
"""
return db.Key.from_path(cls.kind(), str(mapreduce_id))
@classmethod
def get_by_job_id(cls, mapreduce_id):
"""Retrieves the instance of state for a Job.
Args:
mapreduce_id: The mapreduce job to retrieve.
Returns:
instance of MapreduceState for passed id.
"""
return db.get(cls.get_key_by_job_id(mapreduce_id))
def set_processed_counts(self, shards_processed):
"""Updates a chart url to display processed count for each shard.
Args:
shards_processed: list of integers with number of processed entities in
each shard
"""
chart = google_chart_api.BarChart(shards_processed)
shard_count = len(shards_processed)
if shards_processed:
# Only 16 labels on the whole chart.
stride_length = max(1, shard_count / 16)
chart.bottom.labels = []
for x in xrange(shard_count):
if (x % stride_length == 0 or
x == shard_count - 1):
chart.bottom.labels.append(x)
else:
chart.bottom.labels.append("")
chart.left.labels = ["0", str(max(shards_processed))]
chart.left.min = 0
self.chart_width = min(700, max(300, shard_count * 20))
self.chart_url = chart.display.Url(self.chart_width, 200)
def get_processed(self):
"""Number of processed entities.
Returns:
The total number of processed entities as int.
"""
return self.counters_map.get(context.COUNTER_MAPPER_CALLS)
processed = property(get_processed)
@staticmethod
def create_new(mapreduce_id=None,
gettime=datetime.datetime.now):
"""Create a new MapreduceState.
Args:
mapreduce_id: Mapreduce id as string.
gettime: Used for testing.
"""
if not mapreduce_id:
mapreduce_id = MapreduceState.new_mapreduce_id()
state = MapreduceState(key_name=mapreduce_id,
last_poll_time=gettime())
state.set_processed_counts([])
return state
@staticmethod
def new_mapreduce_id():
"""Generate new mapreduce id."""
return util._get_descending_key()
def __eq__(self, other):
if not isinstance(other, self.__class__):
return False
return self.properties() == other.properties()
class NoninflectedPage(search.Searchable, db.Model):
"""Used to test search without stemming, e.g. for precise, non-inflected words"""
author_name = db.StringProperty()
content = db.TextProperty()
INDEX_STEMMING = False
INDEX_ONLY = ['content']
class PostDump(db.Model):
handle = db.StringProperty(required=True)
json = db.TextProperty()
group = db.StringProperty()
touch_time = db.DateTimeProperty(auto_now_add=True, auto_now=True)
