class Greeting(db.Model):
author = db.UserProperty()
content = db.StringProperty(multiline=True)
date = db.DateTimeProperty(auto_now_add=True)
class Possession(db.Model):
flowerbed = db.ReferenceProperty(reference_class=Flowerbed)
lost = db.BooleanProperty(default=False)
timestamp = db.DateTimeProperty(indexed=False, auto_now=True)
class Rule(db.Model):
name = db.StringProperty(indexed=False)
value = db.IntegerProperty(indexed=False)
timestamp = db.DateTimeProperty(indexed=False, auto_now=True)
class BlogEntry(db.Model):
''' Blog post datastore model.
Includes totals for likes, dislikes to decrease datastore queries to
Likes model (e.g. on blog index pages with many blog scores).
Attributes:
subject: Subject of the post.
content: Content of the post.
tags: keywords to describe comment
created: Automatically generated DateTime when the post was submitted.
last_modified: Automatically generated DateTime when the post was updated.
author: User name of the author of the post.
author_id: User id of the author of the post.
close_comments: no new comments allowed
hide_comments: do not show comments to the public
auth_comments: if True, no need to authorise each comment.
sum_comments: Number of comments submitted for this post.
sum_unauth_comments: Number of unauthorised comments submitted for this post.
sum_likes: Number of likes submitted for this post.
sum_dislikes: Number of dislikes submitted for this post.
'''
subject = db.StringProperty(required=True)
content = db.TextProperty(required=True)
tags = db.StringListProperty()
created = db.DateTimeProperty(auto_now_add=True)
last_modified = db.DateTimeProperty(auto_now=True)
author = db.StringProperty(required=True)
author_id = db.IntegerProperty(required=True)
close_comments = db.BooleanProperty(default=False)
hide_comments = db.BooleanProperty(default=False)
auth_comments = db.BooleanProperty(default=False)
sum_comments = db.IntegerProperty(default=0)
sum_unauth_comments = db.IntegerProperty(default=0)
sum_likes = db.IntegerProperty(default=0)
sum_dislikes = db.IntegerProperty(default=0)
@classmethod
def by_id(cls, pid):
return BlogEntry.get_by_id(int(pid), parent=blog_key())
def add_comment(self, userName, uid, comment):
k = self.key()
new_comment = Comment(parent=blog_key(),
commenter_username=userName,
commenter_userid=uid,
comment=comment,
authorised=self.auth_comments,
post_id=k.id())
c = new_comment.put()
if self.auth_comments == True:
self.sum_comments += 1
else:
self.sum_unauth_comments += 1
k = self.put()
return k.id()
def get_comments(self, cid=None):
if cid:
q = Comment.by_id(cid)
else:
q = Comment.all()
q.ancestor(blog_key())
q.fetch(ITEMS_PER_PAGE)
return q
class Post(db.Model):
subject = db.StringProperty(required=True)
content = db.TextProperty(required=True)
created = db.DateTimeProperty(auto_now_add=True)
class UserEvent(db.Expando):
date = db.DateTimeProperty(auto_now_add=True)
ev_type = db.IntegerProperty(required=True, indexed=True)
requester = db.StringProperty(required=True, indexed=True)
class AsciiArt(db.Model):
art = db.TextProperty(required = True)
posted = db.DateTimeProperty(auto_now_add = True)
title = db.StringProperty(required = True)
class UsedNonce(db.Model):
"""An OpenID nonce that has been used.
"""
server_url = db.LinkProperty()
timestamp = db.DateTimeProperty()
salt = db.StringProperty()
class PurchaseLogs(db.Model):
purchaseTimeStamp = db.DateTimeProperty(required= True, indexed= True)
userId = db.StringProperty(required=True, indexed= True)
productId = db.StringProperty(required=True, indexed = False)
eventType = db.StringProperty(required= True, indexed= False)
class ShardState(db.Model):
"""Single shard execution state.
The shard state is stored in the datastore and is later aggregated by
controller task. Shard key_name is equal to shard_id.
Properties:
active: if we have this shard still running as boolean.
counters_map: shard's counters map as CountersMap. Mirrors
counters_map_json.
mapreduce_id: unique id of the mapreduce.
shard_id: unique id of this shard as string.
shard_number: ordered number for this shard.
result_status: If not None, the final status of this shard.
update_time: The last time this shard state was updated.
shard_description: A string description of the work this shard will do.
last_work_item: A string description of the last work item processed.
writer_state: writer state for this shard. This is filled when a job
has one output per shard by OutputWriter's create method.
slice_id: slice id of current executing slice. A task
will not run unless its slice_id matches this. Initial
value is 0. By the end of slice execution, this number is
incremented by 1.
slice_start_time: a slice updates this to now at the beginning of
execution transactionally. If transaction succeeds, the current task holds
a lease of slice duration + some grace period. During this time, no
other task with the same slice_id will execute. Upon slice failure,
the task should try to unset this value to allow retries to carry on
ASAP. slice_start_time is only meaningful when slice_id is the same.
slice_request_id: the request id that holds/held the lease. When lease has
expired, new request needs to verify that said request has indeed
ended according to logs API. Do this only when lease has expired
because logs API is expensive. This field should always be set/unset
with slice_start_time. It is possible Logs API doesn't log a request
at all or doesn't log the end of a request. So a new request can
proceed after a long conservative timeout.
slice_retries: the number of times a slice has been retried due to
data processing error (non taskqueue/datastore). This count is
only a lower bound and is used to determined when to fail a slice
completely.
"""
RESULT_SUCCESS = "success"
RESULT_FAILED = "failed"
# Shard can be in aborted state when user issued abort, or controller
# issued abort because some other shard failed.
RESULT_ABORTED = "aborted"
_RESULTS = frozenset([RESULT_SUCCESS, RESULT_FAILED, RESULT_ABORTED])
# Functional properties.
active = db.BooleanProperty(default=True, indexed=False)
counters_map = JsonProperty(CountersMap, default=CountersMap(), indexed=False)
result_status = db.StringProperty(choices=_RESULTS, indexed=False)
retries = db.IntegerProperty(default=0, indexed=False)
writer_state = JsonProperty(dict, indexed=False)
slice_id = db.IntegerProperty(default=0, indexed=False)
slice_start_time = db.DateTimeProperty(indexed=False)
slice_request_id = db.ByteStringProperty(indexed=False)
slice_retries = db.IntegerProperty(default=0, indexed=False)
# For UI purposes only.
mapreduce_id = db.StringProperty(required=True)
update_time = db.DateTimeProperty(auto_now=True, indexed=False)
shard_description = db.TextProperty(default="")
last_work_item = db.TextProperty(default="")
def __str__(self):
kv = {"active": self.active,
"slice_id": self.slice_id,
"last_work_item": self.last_work_item,
"update_time": self.update_time}
if self.result_status:
kv["result_status"] = self.result_status
if self.retries:
kv["retries"] = self.retries
if self.slice_start_time:
kv["slice_start_time"] = self.slice_start_time
if self.slice_retries:
kv["slice_retries"] = self.slice_retries
if self.slice_request_id:
kv["slice_request_id"] = self.slice_request_id
keys = kv.keys()
keys.sort()
result = "ShardState is {"
for k in keys:
result += k + ":" + str(kv[k]) + ","
result += "}"
return result
def reset_for_retry(self):
"""Reset self for shard retry."""
self.retries += 1
self.last_work_item = ""
self.active = True
self.result_status = None
self.counters_map = CountersMap()
self.slice_id = 0
self.slice_start_time = None
self.slice_request_id = None
self.slice_retries = 0
def advance_for_next_slice(self):
"""Advance self for next slice."""
self.slice_id += 1
self.slice_start_time = None
self.slice_request_id = None
self.slice_retries = 0
def copy_from(self, other_state):
"""Copy data from another shard state entity to self."""
for prop in self.properties().values():
setattr(self, prop.name, getattr(other_state, prop.name))
def get_shard_number(self):
"""Gets the shard number from the key name."""
return int(self.key().name().split("-")[-1])
shard_number = property(get_shard_number)
def get_shard_id(self):
"""Returns the shard ID."""
return self.key().name()
shard_id = property(get_shard_id)
@classmethod
def kind(cls):
"""Returns entity kind."""
return "_AE_MR_ShardState"
@classmethod
def shard_id_from_number(cls, mapreduce_id, shard_number):
"""Get shard id by mapreduce id and shard number.
Args:
mapreduce_id: mapreduce id as string.
shard_number: shard number to compute id for as int.
Returns:
shard id as string.
"""
return "%s-%d" % (mapreduce_id, shard_number)
@classmethod
def get_key_by_shard_id(cls, shard_id):
"""Retrieves the Key for this ShardState.
Args:
shard_id: The shard ID to fetch.
Returns:
The Datatore key to use to retrieve this ShardState.
"""
return db.Key.from_path(cls.kind(), shard_id)
@classmethod
def get_by_shard_id(cls, shard_id):
"""Get shard state from datastore by shard_id.
Args:
shard_id: shard id as string.
Returns:
ShardState for given shard id or None if it's not found.
"""
return cls.get_by_key_name(shard_id)
@classmethod
def find_by_mapreduce_state(cls, mapreduce_state):
"""Find all shard states for given mapreduce.
Args:
mapreduce_state: MapreduceState instance
Returns:
iterable of all ShardState for given mapreduce.
"""
keys = cls.calculate_keys_by_mapreduce_state(mapreduce_state)
return [state for state in db.get(keys) if state]
@classmethod
def calculate_keys_by_mapreduce_state(cls, mapreduce_state):
"""Calculate all shard states keys for given mapreduce.
Args:
mapreduce_state: MapreduceState instance
Returns:
A list of keys for shard states. The corresponding shard states
may not exist.
"""
keys = []
for i in range(mapreduce_state.mapreduce_spec.mapper.shard_count):
shard_id = cls.shard_id_from_number(mapreduce_state.key().name(), i)
keys.append(cls.get_key_by_shard_id(shard_id))
return keys
@classmethod
def find_by_mapreduce_id(cls, mapreduce_id):
logging.error(
"ShardState.find_by_mapreduce_id method may be inconsistent. " +
"ShardState.find_by_mapreduce_state should be used instead.")
return cls.all().filter(
"mapreduce_id =", mapreduce_id).fetch(99999)
@classmethod
def create_new(cls, mapreduce_id, shard_number):
"""Create new shard state.
Args:
mapreduce_id: unique mapreduce id as string.
shard_number: shard number for which to create shard state.
Returns:
new instance of ShardState ready to put into datastore.
"""
shard_id = cls.shard_id_from_number(mapreduce_id, shard_number)
state = cls(key_name=shard_id,
mapreduce_id=mapreduce_id)
return state
class MapreduceState(db.Model):
"""Holds accumulated state of mapreduce execution.
MapreduceState is stored in datastore with a key name equal to the
mapreduce ID. Only controller tasks can write to MapreduceState.
Properties:
mapreduce_spec: cached deserialized MapreduceSpec instance. read-only
active: if we have this mapreduce running right now
last_poll_time: last time controller job has polled this mapreduce.
counters_map: shard's counters map as CountersMap. Mirrors
counters_map_json.
chart_url: last computed mapreduce status chart url. This chart displays the
progress of all the shards the best way it can.
sparkline_url: last computed mapreduce status chart url in small format.
result_status: If not None, the final status of the job.
active_shards: How many shards are still processing.
start_time: When the job started.
writer_state: Json property to be used by writer to store its state.
This is filled when single output per job. Will be deprecated.
"""
RESULT_SUCCESS = "success"
RESULT_FAILED = "failed"
RESULT_ABORTED = "aborted"
_RESULTS = frozenset([RESULT_SUCCESS, RESULT_FAILED, RESULT_ABORTED])
# Functional properties.
mapreduce_spec = JsonProperty(MapreduceSpec, indexed=False)
active = db.BooleanProperty(default=True, indexed=False)
last_poll_time = db.DateTimeProperty(required=True)
counters_map = JsonProperty(CountersMap, default=CountersMap(), indexed=False)
app_id = db.StringProperty(required=False, indexed=True)
writer_state = JsonProperty(dict, indexed=False)
# For UI purposes only.
chart_url = db.TextProperty(default="")
chart_width = db.IntegerProperty(default=300, indexed=False)
sparkline_url = db.TextProperty(default="")
result_status = db.StringProperty(required=False, choices=_RESULTS)
active_shards = db.IntegerProperty(default=0, indexed=False)
failed_shards = db.IntegerProperty(default=0, indexed=False)
aborted_shards = db.IntegerProperty(default=0, indexed=False)
start_time = db.DateTimeProperty(auto_now_add=True)
@classmethod
def kind(cls):
"""Returns entity kind."""
return "_AE_MR_MapreduceState"
@classmethod
def get_key_by_job_id(cls, mapreduce_id):
"""Retrieves the Key for a Job.
Args:
mapreduce_id: The job to retrieve.
Returns:
Datastore Key that can be used to fetch the MapreduceState.
"""
return db.Key.from_path(cls.kind(), str(mapreduce_id))
@classmethod
def get_by_job_id(cls, mapreduce_id):
"""Retrieves the instance of state for a Job.
Args:
mapreduce_id: The mapreduce job to retrieve.
Returns:
instance of MapreduceState for passed id.
"""
return db.get(cls.get_key_by_job_id(mapreduce_id))
def set_processed_counts(self, shards_processed):
"""Updates a chart url to display processed count for each shard.
Args:
shards_processed: list of integers with number of processed entities in
each shard
"""
chart = google_chart_api.BarChart(shards_processed)
shard_count = len(shards_processed)
if shards_processed:
# Only 16 labels on the whole chart.
stride_length = max(1, shard_count / 16)
chart.bottom.labels = []
for x in xrange(shard_count):
if (x % stride_length == 0 or
x == shard_count - 1):
chart.bottom.labels.append(x)
else:
chart.bottom.labels.append("")
chart.left.labels = ['0', str(max(shards_processed))]
chart.left.min = 0
self.chart_width = min(700, max(300, shard_count * 20))
self.chart_url = chart.display.Url(self.chart_width, 200)
def get_processed(self):
"""Number of processed entities.
Returns:
The total number of processed entities as int.
"""
return self.counters_map.get(context.COUNTER_MAPPER_CALLS)
processed = property(get_processed)
@staticmethod
def create_new(mapreduce_id=None,
gettime=datetime.datetime.now):
"""Create a new MapreduceState.
Args:
mapreduce_id: Mapreduce id as string.
gettime: Used for testing.
"""
if not mapreduce_id:
mapreduce_id = MapreduceState.new_mapreduce_id()
state = MapreduceState(key_name=mapreduce_id,
last_poll_time=gettime())
state.set_processed_counts([])
return state
@staticmethod
def new_mapreduce_id():
"""Generate new mapreduce id."""
return _get_descending_key()
class Blog(db.Model):
title = db.StringProperty(required=True)
entry_blog = db.TextProperty(required=True)
created = db.DateTimeProperty(auto_now_add=True)
class Page(db.Model):
url = db.StringProperty(required=True)
content = db.TextProperty(required=True)
created = db.DateTimeProperty(auto_now_add=True)
class ToDoModel(db.Model): description = db.StringProperty() created = db.DateTimeProperty(auto_now_add=True)
class Urly(db.Model):
"""Our one-and-only model"""
href = db.LinkProperty(required=True)
created_at = db.DateTimeProperty(auto_now_add=True)
KEY_BASE = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
BASE = 62
def code(self):
"""Return our code, our base-62 encoded id"""
if not self.is_saved():
return None
nid = self.key().id()
s = []
while nid:
nid, c = divmod(nid, Urly.BASE)
s.append(Urly.KEY_BASE[c])
s.reverse()
return "".join(s)
def to_json(self):
"""JSON is so simple that we won't worry about a template at this point"""
return "{\"code\":\"%s\",\"href\":\"%s\"}\n" % (self.code(), self.href)
def to_xml(self):
"""Like JSON, XML is simple enough that we won't template now"""
msg = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
msg += "<urly code=\"%s\" href=\"%s\" />\n" % (self.code(),
cgi.escape(self.href))
return msg
def to_text(self):
return "http://ur.ly/%s" % self.code()
def save_in_cache(self):
"""We don't really care if this fails"""
memcache.set(self.code(), self)
@staticmethod
def find_or_create_by_href(href):
query = db.Query(Urly)
query.filter('href =', href)
u = query.get()
if not u:
u = Urly(href=href)
u.put()
u.save_in_cache()
return u
@staticmethod
def code_to_id(code):
aid = 0L
for c in code:
aid *= Urly.BASE
aid += Urly.KEY_BASE.index(c)
return aid
@staticmethod
def find_by_code(code):
try:
u = memcache.get(code)
except:
# http://code.google.com/p/googleappengine/issues/detail?id=417
logging.error("Urly.find_by_code() memcached error")
u = None
if u is not None:
logging.info("Urly.find_by_code() cache HIT: %s", str(code))
return u
logging.info("Urly.find_by_code() cache MISS: %s", str(code))
aid = Urly.code_to_id(code)
try:
u = Urly.get_by_id(int(aid))
if u is not None:
u.save_in_cache()
return u
except db.BadValueError:
return None
class Searches(db.Model):
user_id = db.UserProperty()
query = db.StringProperty(multiline=True)
timestamp = db.DateTimeProperty(auto_now_add=True)
private = db.BooleanProperty()
class _AppEngineUtilities_Cache(db.Model):
# It's up to the application to determine the format of their keys
cachekey = db.StringProperty()
createTime = db.DateTimeProperty(auto_now_add=True)
timeout = db.DateTimeProperty()
value = db.BlobProperty()
class GMAArt(db.Model):
title = db.StringProperty(required=True)
art = db.TextProperty(required=True)
cdate = db.DateTimeProperty(auto_now_add=True)
cords = db.GeoPtProperty()
ip = db.StringProperty()
class deposit(db.Model):
clientid = db.StringProperty(required=True)
created = db.DateTimeProperty(auto_now_add=True)
amountpaid = db.StringProperty(required=True)
class Art(db.Model):
title = db.StringProperty(required=True)
art = db.TextProperty(required=True)
created = db.DateTimeProperty(auto_now_add=True)
class ART(db.Model): title=db.StringProperty() arttype=db.TextProperty() created=db.DateTimeProperty(auto_now_add=True)
class Post(db.Model):
title = db.StringProperty(required=True)
body = db.TextProperty(required=True)
created = db.DateTimeProperty(auto_now_add=True)
# TODO - we need an author field here; it should be required
author = db.StringProperty(required=True)
class Vote(db.Model):
user = db.StringProperty(required=True)
issue = db.ReferenceProperty(Issue, required=True)
timestamp = db.DateTimeProperty(auto_now_add=True)
class Comment(db.Model):
name = db.StringProperty(required=True)
body = db.TextProperty(required=True)
created = db.DateTimeProperty(auto_now_add=True)
class FlowerbedLog(db.Model):
flowerbed = db.ReferenceProperty(indexed=False, reference_class=Flowerbed)
owner = db.StringProperty()
attackPlants = db.IntegerProperty(indexed=False)
defensePlants = db.IntegerProperty(indexed=False)
timestamp = db.DateTimeProperty(auto_now=True)
class BlogPost(db.Model):
title = db.StringProperty(required=True)
thoughts = db.TextProperty(required=True)
created = db.DateTimeProperty(auto_now_add=True)
class Bookmark(db.Model):
owner = db.StringProperty(indexed=False)
flowerbed = db.ReferenceProperty(indexed=False, reference_class=Flowerbed)
timestamp = db.DateTimeProperty(indexed=False, auto_now=True)
class NewEntry(db.Model):
title = db.StringProperty(required=True)
entry = db.TextProperty(required=True)
created = db.DateTimeProperty(auto_now_add=True)
modified = db.DateTimeProperty(auto_now_add=True)
class Backpack(db.Model):
owner = db.StringProperty(indexed=False)
name = db.StringProperty() #Item.name
amount = db.IntegerProperty(indexed=False)
timestamp = db.DateTimeProperty(indexed=False, auto_now=True)
class Comment(db.Model):
comment = db.StringProperty(required=True)
post_id = db.StringProperty(required=True)
made_by = db.StringProperty(required=True)
created_in = db.DateTimeProperty(auto_now=True)
