class Meeting(ndb.Model):
datetime = ndb.DateTimeProperty(auto_now_add=True, required=True)
name = ndb.StringProperty(required=True)
troop = ndb.KeyProperty(kind=Troop, required=True)
duration = ndb.IntegerProperty(default=90, required=True) #minutes
semester = ndb.KeyProperty(kind=Semester, required=False) # TODO: remove
attendingPersons = ndb.KeyProperty(
kind=Person, repeated=True) # list of attending persons' keys
@staticmethod
def __getMemcacheKeyString(troop_key):
return 'tms:' + str(troop_key)
@staticmethod
def getId(meetingDatetime, troop_key):
return meetingDatetime.strftime("%Y%m%d%H%M") + str(troop_key.id())
@staticmethod
def getOrCreate(troop_key, name, datetime, duration):
m = Meeting.get_by_id(Meeting.getId(datetime, troop_key),
use_memcache=True)
if m != None:
if m.name != name or m.duration != duration:
m.name = name
m.duration = duration
m.put()
else:
m = Meeting(id=Meeting.getId(datetime, troop_key),
datetime=datetime,
name=name,
troop=troop_key,
duration=duration)
troopmeeting_keys = memcache.get(
Meeting.__getMemcacheKeyString(troop_key))
if troopmeeting_keys is not None and m.key not in troopmeeting_keys:
troopmeeting_keys.append(m.key)
memcache.replace(Meeting.__getMemcacheKeyString(troop_key),
troopmeeting_keys)
return m
@staticmethod
def gettroopmeetings(troop_key):
troopmeetings = []
troopmeeting_keys = memcache.get(
Meeting.__getMemcacheKeyString(troop_key))
if troopmeeting_keys is None:
troopmeeting_keys = Meeting.query(
Meeting.troop == troop_key).fetch(keys_only=True)
memcache.add(Meeting.__getMemcacheKeyString(troop_key),
troopmeeting_keys)
for tm_key in troopmeeting_keys:
m = tm_key.get()
if m != None:
troopmeetings.append(m)
troopmeetings.sort(key=lambda x: x.datetime, reverse=True)
return troopmeetings
def delete(self):
self.key.delete()
troopmeeting_keys = memcache.get(
Meeting.__getMemcacheKeyString(self.troop))
if troopmeeting_keys is not None:
troopmeeting_keys.remove(self.key)
memcache.replace(Meeting.__getMemcacheKeyString(self.troop),
troopmeeting_keys)
def commit(self):
self.put()
def getdate(self):
return self.datetime.strftime("%Y-%m-%d")
def gettime(self):
return self.datetime.strftime("%H:%M")
def getname(self):
return self.name
def getendtime(self):
maxEndTime = self.datetime.replace(hour=23, minute=59, second=59)
endtime = self.datetime + datetime.timedelta(minutes=self.duration)
if endtime > maxEndTime:
endtime = maxEndTime # limit to the current day (to keep Stop time after Start time)
return endtime.strftime('%H:%M')
class Message(ndb.Model):
message_text = ndb.StringProperty()
message_author = ndb.StringProperty()
message_email = ndb.StringProperty()
deleted = ndb.BooleanProperty(default=False)
created = ndb.DateTimeProperty(auto_now_add=True)
class Job(ndb.Model):
"""A Pinpoint job."""
created = ndb.DateTimeProperty(required=True, auto_now_add=True)
# Don't use `auto_now` for `updated`. When we do data migration, we need
# to be able to modify the Job without changing the Job's completion time.
updated = ndb.DateTimeProperty(required=True, auto_now_add=True)
# The name of the Task Queue task this job is running on. If it's present, the
# job is running. The task is also None for Task Queue retries.
task = ndb.StringProperty()
# The string contents of any Exception that was thrown to the top level.
# If it's present, the job failed.
exception = ndb.TextProperty()
# Request parameters.
arguments = ndb.JsonProperty(required=True)
# TODO: The bug id is only used for posting bug comments when a job starts and
# completes. This probably should not be the responsibility of Pinpoint.
bug_id = ndb.IntegerProperty()
# User-provided name of the job.
name = ndb.StringProperty()
# Email of the job creator.
user = ndb.StringProperty()
# The Gerrit server url and change id of the code review to update upon
# completion.
gerrit_server = ndb.StringProperty()
gerrit_change_id = ndb.StringProperty()
state = ndb.PickleProperty(required=True, compressed=True)
tags = ndb.JsonProperty()
@classmethod
def New(cls, quests, changes, arguments=None, bug_id=None,
comparison_mode=None, gerrit_server=None, gerrit_change_id=None,
name=None, pin=None, tags=None, user=None):
"""Creates a new Job, adds Changes to it, and puts it in the Datstore.
Args:
quests: An iterable of Quests for the Job to run.
changes: An iterable of the initial Changes to run on.
arguments: A dict with the original arguments used to start the Job.
bug_id: A monorail issue id number to post Job updates to.
comparison_mode: Either 'functional' or 'performance', which the Job uses
to figure out whether to perform a functional or performance bisect.
If None, the Job will not automatically add any Attempts or Changes.
gerrit_server: Server of the Gerrit code review to update with job
results.
gerrit_change_id: Change id of the Gerrit code review to update with job
results.
name: The user-provided name of the Job.
pin: A Change (Commits + Patch) to apply to every Change in this Job.
tags: A dict of key-value pairs used to filter the Jobs listings.
user: The email of the Job creator.
Returns:
A Job object.
"""
state = job_state.JobState(quests, comparison_mode=comparison_mode, pin=pin)
job = cls(state=state, arguments=arguments or {},
bug_id=bug_id, gerrit_server=gerrit_server,
gerrit_change_id=gerrit_change_id,
name=name, tags=tags, user=user)
for c in changes:
job.AddChange(c)
job.put()
return job
@property
def job_id(self):
return '%x' % self.key.id()
@property
def status(self):
if self.task:
return 'Running'
if self.exception:
return 'Failed'
return 'Completed'
@property
def url(self):
return 'https://%s/job/%s' % (os.environ['HTTP_HOST'], self.job_id)
def AddChange(self, change):
self.state.AddChange(change)
def Start(self):
"""Starts the Job and updates it in the Datastore.
This method is designed to return fast, so that Job creation is responsive
to the user. It schedules the Job on the task queue without running
anything. It also posts a bug comment, and updates the Datastore.
"""
self._Schedule()
self.put()
title = _ROUND_PUSHPIN + ' Pinpoint job started.'
comment = '\n'.join((title, self.url))
self._PostBugComment(comment, send_email=False)
def _Complete(self):
try:
results2.ScheduleResults2Generation(self)
except taskqueue.Error:
pass
self._FormatAndPostBugCommentOnComplete()
self._UpdateGerritIfNeeded()
def _FormatAndPostBugCommentOnComplete(self):
if not self.state.comparison_mode:
# There is no comparison metric.
title = "<b>%s Job complete. See results below.</b>" % _ROUND_PUSHPIN
self._PostBugComment('\n'.join((title, self.url)))
return
# There is a comparison metric.
differences = tuple(self.state.Differences())
if not differences:
title = "<b>%s Couldn't reproduce a difference.</b>" % _ROUND_PUSHPIN
self._PostBugComment('\n'.join((title, self.url)))
return
# Include list of Changes.
owner = None
sheriff = None
cc_list = set()
difference_details = []
for _, change, values_a, values_b in differences:
if change.patch:
commit_info = change.patch.AsDict()
else:
commit_info = change.last_commit.AsDict()
# TODO: Assign the largest difference, not the last one.
owner = commit_info['author']
sheriff = utils.GetSheriffForAutorollCommit(commit_info)
cc_list.add(commit_info['author'])
difference = _FormatDifferenceForBug(commit_info, values_a, values_b)
difference_details.append(difference)
# Header.
if len(differences) == 1:
status = 'Found a significant difference after 1 commit.'
else:
status = ('Found significant differences after each of %d commits.' %
len(differences))
title = '<b>%s %s</b>' % (_ROUND_PUSHPIN, status)
header = '\n'.join((title, self.url))
# Body.
body = '\n\n'.join(difference_details)
if sheriff:
owner = sheriff
body += '\n\nAssigning to sheriff %s because "%s" is a roll.' % (
sheriff, commit_info['subject'])
# Footer.
footer = ('Understanding performance regressions:\n'
' http://g.co/ChromePerformanceRegressions')
# Bring it all together.
comment = '\n\n'.join((header, body, footer))
current_bug_status = self._GetBugStatus()
if (not current_bug_status or
current_bug_status in ['Untriaged', 'Unconfirmed', 'Available']):
# Set the bug status and owner if this bug is opened and unowned.
self._PostBugComment(comment, status='Assigned',
cc_list=sorted(cc_list), owner=owner)
else:
# Only update the comment and cc list if this bug is assigned or closed.
self._PostBugComment(comment, cc_list=sorted(cc_list))
def _UpdateGerritIfNeeded(self):
if self.gerrit_server and self.gerrit_change_id:
gerrit_service.PostChangeComment(
self.gerrit_server,
self.gerrit_change_id,
'%s Job complete.\n\nSee results at: %s' % (_ROUND_PUSHPIN, self.url))
def Fail(self):
self.exception = traceback.format_exc()
title = _CRYING_CAT_FACE + ' Pinpoint job stopped with an error.'
comment = '\n'.join((title, self.url, '', sys.exc_info()[1].message))
self._PostBugComment(comment)
def _Schedule(self):
# Set a task name to deduplicate retries. This adds some latency, but we're
# not latency-sensitive. If Job.Run() works asynchronously in the future,
# we don't need to worry about duplicate tasks.
# https://github.com/catapult-project/catapult/issues/3900
task_name = str(uuid.uuid4())
try:
task = taskqueue.add(
queue_name='job-queue', url='/api/run/' + self.job_id,
name=task_name, countdown=_TASK_INTERVAL)
except (apiproxy_errors.DeadlineExceededError, taskqueue.TransientError):
task = taskqueue.add(
queue_name='job-queue', url='/api/run/' + self.job_id,
name=task_name, countdown=_TASK_INTERVAL)
self.task = task.name
def Run(self):
"""Runs this Job.
Loops through all Attempts and checks the status of each one, kicking off
tasks as needed. Does not block to wait for all tasks to finish. Also
compares adjacent Changes' results and adds any additional Attempts or
Changes as needed. If there are any incomplete tasks, schedules another
Run() call on the task queue.
"""
self.exception = None # In case the Job succeeds on retry.
self.task = None # In case an exception is thrown.
try:
if self.state.comparison_mode:
self.state.Explore()
work_left = self.state.ScheduleWork()
# Schedule moar task.
if work_left:
self._Schedule()
else:
self._Complete()
except BaseException:
self.Fail()
raise
finally:
# Don't use `auto_now` for `updated`. When we do data migration, we need
# to be able to modify the Job without changing the Job's completion time.
self.updated = datetime.datetime.now()
try:
self.put()
except (datastore_errors.Timeout,
datastore_errors.TransactionFailedError):
# Retry once.
self.put()
except datastore_errors.BadRequestError:
if self.task:
queue = taskqueue.Queue('job-queue')
queue.delete_tasks(taskqueue.Task(name=self.task))
self.task = None
# The _JobState is too large to fit in an ndb property.
# Load the Job from before we updated it, and fail it.
job = self.key.get(use_cache=False)
job.task = None
job.Fail()
job.updated = datetime.datetime.now()
job.put()
raise
def AsDict(self, options=None):
d = {
'job_id': self.job_id,
'arguments': self.arguments,
'bug_id': self.bug_id,
'name': self.name,
'user': self.user,
'created': self.created.isoformat(),
'updated': self.updated.isoformat(),
'exception': self.exception,
'status': self.status,
}
if not options:
return d
if OPTION_STATE in options:
d.update(self.state.AsDict())
if OPTION_TAGS in options:
d['tags'] = {'tags': self.tags}
return d
def _PostBugComment(self, *args, **kwargs):
if not self.bug_id:
return
issue_tracker = issue_tracker_service.IssueTrackerService(
utils.ServiceAccountHttp())
issue_tracker.AddBugComment(self.bug_id, *args, **kwargs)
def _GetBugStatus(self):
if not self.bug_id:
return None
issue_tracker = issue_tracker_service.IssueTrackerService(
utils.ServiceAccountHttp())
issue_data = issue_tracker.GetIssue(self.bug_id)
return issue_data.get('status')
class BaseCommitLogEntryModel(BaseModel):
"""Base Model for the models that store the log of commits to a
construct.
"""
# Update superclass model to make these properties indexed.
created_on = ndb.DateTimeProperty(auto_now_add=True, indexed=True)
last_updated = ndb.DateTimeProperty(auto_now=True, indexed=True)
# The id of the user.
user_id = ndb.StringProperty(indexed=True, required=True)
# The username of the user, at the time of the edit.
username = ndb.StringProperty(indexed=True, required=True)
# The type of the commit: 'create', 'revert', 'edit', 'delete'.
commit_type = ndb.StringProperty(indexed=True, required=True)
# The commit message.
commit_message = ndb.TextProperty(indexed=False)
# The commit_cmds dict for this commit.
commit_cmds = ndb.JsonProperty(indexed=False, required=True)
# The status of the entity after the edit event ('private', 'public').
post_commit_status = ndb.StringProperty(indexed=True, required=True)
# Whether the entity is community-owned after the edit event.
post_commit_community_owned = ndb.BooleanProperty(indexed=True)
# Whether the entity is private after the edit event. Having a
# separate field for this makes queries faster, since an equality query
# on this property is faster than an inequality query on
# post_commit_status.
post_commit_is_private = ndb.BooleanProperty(indexed=True)
# The version number of the model after this commit.
version = ndb.IntegerProperty()
@classmethod
def create(cls, entity_id, version, committer_id, committer_username,
commit_type, commit_message, commit_cmds, status,
community_owned):
"""This method returns an instance of the CommitLogEntryModel for a
construct with the common fields filled.
Args:
entity_id: str. The ID of the construct corresponding to this
commit log entry model (e.g. the exp_id for an exploration,
the story_id for a story, etc.).
version: int. The version number of the model after the commit.
committer_id: str. The user_id of the user who committed the
change.
committer_username: str. The username of the user who committed the
change.
commit_type: str. The type of commit. Possible values are in
core.storage.base_models.COMMIT_TYPE_CHOICES.
commit_message: str. The commit description message.
commit_cmds: list(dict). A list of commands, describing changes
made in this model, which should give sufficient information to
reconstruct the commit. Each dict always contains:
cmd: str. Unique command.
and then additional arguments for that command.
status: str. The status of the entity after the commit.
community_owned: bool. Whether the entity is community_owned after
the commit.
Returns:
CommitLogEntryModel. Returns the respective CommitLogEntryModel
instance of the construct from which this is called.
"""
return cls(id=cls._get_instance_id(entity_id, version),
user_id=committer_id,
username=committer_username,
commit_type=commit_type,
commit_message=commit_message,
commit_cmds=commit_cmds,
version=version,
post_commit_status=status,
post_commit_community_owned=community_owned,
post_commit_is_private=(
status == constants.ACTIVITY_STATUS_PRIVATE))
@classmethod
def _get_instance_id(cls, target_entity_id, version):
"""This method should be implemented in the inherited classes.
Args:
target_entity_id: str. The ID of the construct corresponding to this
commit log entry model (e.g. the exp_id for an exploration,
the story_id for a story, etc.).
version: int. The version number of the model after the commit.
Raises:
NotImplementedError: The method is not overwritten in derived
classes.
"""
raise NotImplementedError
@classmethod
def get_all_commits(cls, page_size, urlsafe_start_cursor):
"""Fetches a list of all the commits sorted by their last updated
attribute.
Args:
page_size: int. The maximum number of entities to be returned.
urlsafe_start_cursor: str or None. If provided, the list of
returned entities starts from this datastore cursor.
Otherwise, the returned entities start from the beginning
of the full list of entities.
Returns:
3-tuple of (results, cursor, more) as described in fetch_page() at:
https://developers.google.com/appengine/docs/python/ndb/queryclass,
where:
results: List of query results.
cursor: str or None. A query cursor pointing to the next
batch of results. If there are no more results, this might
be None.
more: bool. If True, there are (probably) more results after
this batch. If False, there are no further results after
this batch.
"""
return cls._fetch_page_sorted_by_last_updated(cls.query(), page_size,
urlsafe_start_cursor)
@classmethod
def get_commit(cls, target_entity_id, version):
"""Returns the commit corresponding to an instance id and
version number.
Args:
target_entity_id: str. The ID of the construct corresponding to this
commit log entry model (e.g. the exp_id for an exploration,
the story_id for a story, etc.).
version: int. The version number of the instance
after the commit.
Returns:
BaseCommitLogEntryModel. The commit with the target entity id and
version number.
"""
commit_id = cls._get_instance_id(target_entity_id, version)
return cls.get_by_id(commit_id)
class ClassifierTrainingJobModel(base_models.BaseModel):
"""Model for storing classifier training jobs.
The id of instances of this class has the form
{{exp_id}}.{{random_hash_of_12_chars}}
"""
# The ID of the algorithm used to create the model.
algorithm_id = ndb.StringProperty(required=True,
choices=ALGORITHM_CHOICES,
indexed=True)
# The ID of the interaction to which the algorithm belongs.
interaction_id = ndb.StringProperty(required=True, indexed=True)
# The exploration_id of the exploration to whose state the model belongs.
exp_id = ndb.StringProperty(required=True, indexed=True)
# The exploration version at the time this training job was created.
exp_version = ndb.IntegerProperty(required=True, indexed=True)
# The name of the state to which the model belongs.
state_name = ndb.StringProperty(required=True, indexed=True)
# The status of the training job. It can be either NEW, COMPLETE or PENDING.
status = ndb.StringProperty(required=True,
choices=feconf.ALLOWED_TRAINING_JOB_STATUSES,
default=feconf.TRAINING_JOB_STATUS_PENDING,
indexed=True)
# The training data which is to be populated when retrieving the job.
# The list contains dicts where each dict represents a single training
# data group.
training_data = ndb.JsonProperty(default=None)
# The time when the job's status should next be checked.
# It is incremented by TTL when a job with status NEW is picked up by VM.
next_scheduled_check_time = ndb.DateTimeProperty(required=True,
indexed=True)
# The classifier data which will be populated when storing the results of
# the job.
classifier_data = ndb.JsonProperty(default=None)
# The schema version for the data that is being classified.
data_schema_version = ndb.IntegerProperty(required=True, indexed=True)
@classmethod
def _generate_id(cls, exp_id):
"""Generates a unique id for the training job of the form
{{exp_id}}.{{random_hash_of_16_chars}}
Args:
exp_id: str. ID of the exploration.
Returns:
ID of the new ClassifierTrainingJobModel instance.
Raises:
Exception: The id generator for ClassifierTrainingJobModel is
producing too many collisions.
"""
for _ in python_utils.RANGE(base_models.MAX_RETRIES):
new_id = '%s.%s' % (exp_id,
utils.convert_to_hash(
python_utils.UNICODE(
utils.get_random_int(
base_models.RAND_RANGE)),
base_models.ID_LENGTH))
if not cls.get_by_id(new_id):
return new_id
raise Exception(
'The id generator for ClassifierTrainingJobModel is producing '
'too many collisions.')
@classmethod
def create(cls, algorithm_id, interaction_id, exp_id, exp_version,
next_scheduled_check_time, training_data, state_name, status,
classifier_data, data_schema_version):
"""Creates a new ClassifierTrainingJobModel entry.
Args:
algorithm_id: str. ID of the algorithm used to generate the model.
interaction_id: str. ID of the interaction to which the algorithm
belongs.
exp_id: str. ID of the exploration.
exp_version: int. The exploration version at the time
this training job was created.
next_scheduled_check_time: datetime.datetime. The next scheduled
time to check the job.
training_data: dict. The data used in training phase.
state_name: str. The name of the state to which the classifier
belongs.
status: str. The status of the training job.
classifier_data: dict|None. The data stored as result of training.
data_schema_version: int. The schema version for the data.
Returns:
ID of the new ClassifierModel entry.
Raises:
Exception: A model with the same ID already exists.
"""
instance_id = cls._generate_id(exp_id)
training_job_instance = cls(
id=instance_id,
algorithm_id=algorithm_id,
interaction_id=interaction_id,
exp_id=exp_id,
exp_version=exp_version,
next_scheduled_check_time=next_scheduled_check_time,
state_name=state_name,
status=status,
training_data=training_data,
classifier_data=classifier_data,
data_schema_version=data_schema_version)
training_job_instance.put()
return instance_id
@classmethod
def query_new_and_pending_training_jobs(cls, cursor=None):
"""Gets the next 10 jobs which are either in status "new" or "pending",
ordered by their next_scheduled_check_time attribute.
Args:
cursor: str or None. The list of returned entities starts from this
datastore cursor.
Returns:
List of the ClassifierTrainingJobModels with status new or pending.
"""
query = cls.query(
cls.status.IN([
feconf.TRAINING_JOB_STATUS_NEW,
feconf.TRAINING_JOB_STATUS_PENDING
])).filter(cls.next_scheduled_check_time <= (
datetime.datetime.utcnow())).order(
cls.next_scheduled_check_time, cls._key)
job_models, cursor, more = query.fetch_page(10, start_cursor=cursor)
return job_models, cursor, more
@classmethod
def create_multi(cls, job_dicts_list):
"""Creates multiple new ClassifierTrainingJobModel entries.
Args:
job_dicts_list: list(dict). The list of dicts where each dict
represents the attributes of one ClassifierTrainingJobModel.
Returns:
list(str). List of job IDs.
"""
job_models = []
job_ids = []
for job_dict in job_dicts_list:
instance_id = cls._generate_id(job_dict['exp_id'])
training_job_instance = cls(
id=instance_id,
algorithm_id=job_dict['algorithm_id'],
interaction_id=job_dict['interaction_id'],
exp_id=job_dict['exp_id'],
exp_version=job_dict['exp_version'],
next_scheduled_check_time=job_dict[
'next_scheduled_check_time'],
state_name=job_dict['state_name'],
status=job_dict['status'],
training_data=job_dict['training_data'],
classifier_data=job_dict['classifier_data'],
data_schema_version=job_dict['data_schema_version'])
job_models.append(training_job_instance)
job_ids.append(instance_id)
cls.put_multi(job_models)
return job_ids
class Event(ndb.Model):
tags = ndb.StringProperty(repeated=True)
time = ndb.DateTimeProperty()
data = ndb.TextProperty()
create_time = ndb.DateTimeProperty(auto_now_add=True)
class ApnsToken(ndb.Model):
apns_token = ndb.StringProperty(indexed=True)
enabled = ndb.BooleanProperty(indexed=True, default=True)
registration_date = ndb.DateTimeProperty(indexed=False, auto_now_add=True)
class Subscription(ndb.Model):
name = ndb.StringProperty(required=True)
cpf = ndb.StringProperty(required=True)
email = ndb.StringProperty(required=True)
creation = ndb.DateTimeProperty(auto_now_add=True)
class ShopifyUser(User):
domain = ndb.StringProperty()
myshopifyDomain = ndb.StringProperty()
authState = ndb.StringProperty()
authToken = ndb.StringProperty()
pageId = ndb.StringProperty()
facebookPageName = ndb.StringProperty()
facebookToken = ndb.StringProperty()
facebookPageToken = ndb.StringProperty()
facebookPageHandle = ndb.StringProperty()
tokenExpiresIn = ndb.DateTimeProperty()
welcomeMessage = ndb.StringProperty()
requestAttribute = 'shopifyUser'
sessionKey = 'shopifyUserKey'
@classmethod
def forShop(cls, myshopifyDomain):
return ShopifyUser.query().filter(
ShopifyUser.myshopifyDomain == \
myshopifyDomain
).get()
@classmethod
def forPageId(cls, pageId):
if pageId:
return ShopifyUser.query().filter(
ShopifyUser.pageId == \
str(pageId)
).get()
def activateSession(self):
if self.authToken == None:
raise Exception(
"Can't activate a shopify session without an authToken")
session = shopify.Session(self.myshopifyDomain, self.authToken)
shopify.ShopifyResource.activate_session(session)
def getAdminUrl(self):
return "/admin/shopify-users/%s/" % self.key.urlsafe()
def getBuyLink(self, variantId, quantity=1):
return "%s/cart/%s:%s?utm_source=FanCommerce" % (
self.myshopifyDomain,
variantId,
quantity,
)
def getViewLink(self, handle, variantId=None):
url = "http://%s/products/%s?utm_source=FanCommerce" % (
self.myshopifyDomain,
handle,
)
if variantId:
url += ("?variant=" + str(variantId))
return url
def getFacebookPages(self):
return api.facebook(
"/me/accounts",
method="GET",
body={
'is_promotable': True,
},
accessToken=self.facebookToken,
)
def _getProductsQuery(self):
return ShopifyProduct.query()\
.filter(ShopifyProduct.shopifyUser == self.key)\
def _getConversationsQuery(self):
return ShopifyConversation.query()\
.filter(ShopifyConversation.shopifyUser == self.key)\
def getProducts(self):
return self._getProductsQuery().fetch()
def getConversations(self):
return self._getConversationsQuery()\
.order(-ShopifyConversation.created).fetch()
def getProductCount(self):
return self._getProductsQuery().count()
def getConversationCount(self):
return self._getConversationsQuery().count()
def getJson(self):
return {
'pageId': self.pageId,
'facebookToken': self.facebookToken,
'facebookPageName': self.facebookPageName,
'facebookPageHandle': self.facebookPageHandle,
'welcomeMessage': self.welcomeMessage,
}
def getPopularProducts(self):
return ShopifyProduct.query()\
.filter(ShopifyProduct.shopifyUser == self.key)\
.order(-ShopifyProduct.orderCount)\
.fetch(limit=10)
def search(self, term):
index = search.Index(self.myshopifyDomain)
results = index.search(query=search.Query(
term,
options=search.QueryOptions(limit=10, cursor=search.Cursor())))
keys = []
for result in results.results:
keys.append(ndb.Key(ShopifyProduct, int(result.doc_id)))
products = ndb.get_multi(keys)
return products
class Account(ndb.Model):
email = ndb.StringProperty()
expenses = ndb.StringProperty(repeated=True)
created = ndb.DateTimeProperty(auto_now_add=True)
class candidate(ndb.Model):
born = ndb.DateTimeProperty(auto_now_add=True)
vote = ndb.IntegerProperty()
name = ndb.StringProperty()
class News(ndb.Model):
"""Class to store news items in GAE NDB"""
title = ndb.StringProperty(required=True)
date = ndb.DateTimeProperty(required=True)
create_date = ndb.DateTimeProperty(auto_now=True)
link = ndb.StringProperty(required=True)
author = ndb.StringProperty()
text = ndb.TextProperty()
news_type = ndb.StringProperty(required=True)
def add_db_unique(self, entry):
""" Given an entry dict, add it to ndb. Make sure its unique."""
if entry is None:
raise InputError("Entry", entry, "Entry cannot be None.")
title = entry['title']
link = entry['link']
author = entry['author']
text = entry['text']
ptime = entry['print_time']
ndate = entry['datetime']
dt = datetime.fromtimestamp(mktime(ndate))
typ = entry['type']
ky = hashlib.md5(title).hexdigest()
new_key = ndb.Key(News, ky)
entry = new_key.get()
if entry is None:
logging.debug("News entry not present. Adding it to db")
news = News(title=title,
date=dt,
link=link,
author=author,
text=text,
news_type=typ)
news.key = new_key
news.put()
return True
else:
logging.debug("News entry already present.")
if entry.date == dt:
logging.debug("News entry has same date. Skipping.")
return False
else:
logging.debug("News entry has new date. Updating.")
entry.date = dt
entry.put()
return True
def get_entry_key(self, ky):
"""For the given key, return the corresponding entry from ndb."""
if ky is None:
raise InputError("Key", ky, "Key cannot be None.")
n_key = ndb.Key(News, ky)
entry = n_key.get()
if entry is None:
logging.error("No entry for given key.")
return None
else:
return entry
def get_key(self):
"""Return the key of this entry."""
if self.title is None:
logging.debug("Unable to generate key due to None entry.")
return None
else:
return hashlib.md5(self.title).hexdigest()
class Replay(ndb.Model):
"""Models an individual Guestbook entry."""
tag = ndb.StringProperty()
shortURL = ndb.StringProperty()
date = ndb.DateTimeProperty(auto_now_add=True)
class ExampleModel(ndb.Model):
"""Example Model"""
example_name = ndb.StringProperty(required=True)
example_description = ndb.TextProperty(required=True)
added_by = ndb.UserProperty()
timestamp = ndb.DateTimeProperty(auto_now_add=True)
class BasicModel(ndb.Model):
created_at = ndb.DateTimeProperty(auto_now_add=True)
created_by = ndb.UserProperty(auto_current_user_add=True)
modified_at = ndb.DateTimeProperty(auto_now=True)
modified_by = ndb.UserProperty(auto_current_user=True)
class BaseModel(ndb.Model):
__abstract__ = True
lastUpdate = ndb.DateTimeProperty(auto_now=True)
created = ndb.DateTimeProperty(auto_now_add=True)
class Recovery(ndb.Model):
code = ndb.IntegerProperty()
time = ndb.DateTimeProperty(auto_now=True)
class Comment(ndb.Model):
created = ndb.DateTimeProperty(auto_now_add=True)
text = ndb.StringProperty()
author = ndb.StringProperty()
class Message(ndb.Model):
sender = ndb.KeyProperty(kind=User)
text = ndb.TextProperty()
media = ndb.StructuredProperty(
Pair, repeated=True) # pair of content type and link to data
create_time = ndb.DateTimeProperty(auto_now_add=True)
class UserEvent(ndb.Model):
user_id = ndb.StringProperty()
author = ndb.StringProperty()
message = ndb.StringProperty()
date = ndb.DateTimeProperty(auto_now_add=True)
class Greeting(ndb.Model):
"""A main model for representing an individual Guestbook entry."""
author = ndb.StructuredProperty(Author)
content = ndb.StringProperty(indexed=False)
date = ndb.DateTimeProperty(auto_now_add=True)
class SessionData(ndb.Model): subject = ndb.KeyProperty(kind="Subject") views_available = ndb.IntegerProperty(default=10) created_at = ndb.DateTimeProperty(auto_now_add=True) updated_at = ndb.DateTimeProperty(auto_now=True)
class BaseModel(ndb.Model):
"""Base model for all persistent object storage classes."""
# When this entity was first created. This can be overwritten and
# set explicitly.
created_on = ndb.DateTimeProperty(auto_now_add=True, indexed=True)
# When this entity was last updated. This cannot be set directly.
last_updated = ndb.DateTimeProperty(auto_now=True, indexed=True)
# Whether the current version of the model instance is deleted.
deleted = ndb.BooleanProperty(indexed=True, default=False)
@property
def id(self):
"""A unique id for this model instance."""
return self.key.id()
def _pre_put_hook(self):
"""This is run before model instances are saved to the datastore.
Subclasses of BaseModel should override this method.
"""
pass
class EntityNotFoundError(Exception):
"""Raised when no entity for a given id exists in the datastore."""
pass
@classmethod
def get(cls, entity_id, strict=True):
"""Gets an entity by id.
Args:
entity_id: str.
strict: bool. Whether to fail noisily if no entity with the given id
exists in the datastore. Default is True.
Returns:
None, if strict == False and no undeleted entity with the given id
exists in the datastore. Otherwise, the entity instance that
corresponds to the given id.
Raises:
base_models.BaseModel.EntityNotFoundError: if strict == True and
no undeleted entity with the given id exists in the datastore.
"""
entity = cls.get_by_id(entity_id)
if entity and entity.deleted:
entity = None
if strict and entity is None:
raise cls.EntityNotFoundError(
'Entity for class %s with id %s not found' %
(cls.__name__, entity_id))
return entity
@classmethod
def get_multi(cls, entity_ids, include_deleted=False):
"""Gets list of entities by list of ids.
Args:
entity_ids: list(str).
include_deleted: bool. Whether to include deleted entities in the
return list. Default is False.
Returns:
list(*|None). A list that contains model instances that match
the corresponding entity_ids in the input list. If an instance is
not found, or it has been deleted and include_deleted is False,
then the corresponding entry is None.
"""
entity_keys = []
none_argument_indices = []
for index, entity_id in enumerate(entity_ids):
if entity_id:
entity_keys.append(ndb.Key(cls, entity_id))
else:
none_argument_indices.append(index)
entities = ndb.get_multi(entity_keys)
for index in none_argument_indices:
entities.insert(index, None)
if not include_deleted:
for i in xrange(len(entities)):
if entities[i] and entities[i].deleted:
entities[i] = None
return entities
@classmethod
def put_multi(cls, entities):
"""Stores the given ndb.Model instances.
Args:
entities: list(ndb.Model).
"""
ndb.put_multi(entities)
@classmethod
def delete_multi(cls, entities):
"""Deletes the given ndb.Model instances.
Args:
entities: list(ndb.Model).
"""
keys = [entity.key for entity in entities]
ndb.delete_multi(keys)
def delete(self):
"""Deletes this instance."""
super(BaseModel, self).key.delete()
@classmethod
def get_all(cls, include_deleted=False):
"""Gets iterable of all entities of this class.
Args:
include_deleted: bool. If True, then entities that have been marked
deleted are returned as well. Defaults to False.
Returns:
iterable. Filterable iterable of all entities of this class.
"""
query = cls.query()
if not include_deleted:
query = query.filter(cls.deleted == False) # pylint: disable=singleton-comparison
return query
@classmethod
def get_new_id(cls, entity_name):
"""Gets a new id for an entity, based on its name.
The returned id is guaranteed to be unique among all instances of this
entity.
Args:
entity_name: The name of the entity. Coerced to a utf-8 encoded
string. Defaults to ''.
Returns:
str. New unique id for this entity class.
Raises:
Exception: An ID cannot be generated within a reasonable number
of attempts.
"""
try:
entity_name = unicode(entity_name).encode(encoding='utf-8')
except Exception:
entity_name = ''
for _ in range(MAX_RETRIES):
new_id = utils.convert_to_hash(
'%s%s' % (entity_name, utils.get_random_int(RAND_RANGE)),
ID_LENGTH)
if not cls.get_by_id(new_id):
return new_id
raise Exception('New id generator is producing too many collisions.')
@classmethod
def _fetch_page_sorted_by_last_updated(cls, query, page_size,
urlsafe_start_cursor):
"""Fetches a page of entities sorted by their last_updated attribute in
descending order (newly updated first).
Args:
query: ndb.Query.
page_size: int. The maximum number of entities to be returned.
urlsafe_start_cursor: str or None. If provided, the list of returned
entities starts from this datastore cursor. Otherwise,
the returned entities start from the beginning of the full
list of entities.
Returns:
3-tuple of (results, cursor, more) as described in fetch_page() at:
https://developers.google.com/appengine/docs/python/ndb/queryclass,
where:
results: List of query results.
cursor: str or None. A query cursor pointing to the next batch
of results. If there are no more results, this will be None.
more: bool. If True, there are (probably) more results after
this batch. If False, there are no further results after
this batch.
"""
if urlsafe_start_cursor:
start_cursor = datastore_query.Cursor(urlsafe=urlsafe_start_cursor)
else:
start_cursor = None
result = query.order(-cls.last_updated).fetch_page(
page_size, start_cursor=start_cursor)
return (result[0], (result[1].urlsafe() if result[1] else None),
result[2])
class Zip(ndb.Model):
"""Caching for zip code data"""
img_url = ndb.StringProperty()
city = ndb.StringProperty()
state = ndb.StringProperty()
updated = ndb.DateTimeProperty(required=True, auto_now_add=True)
class FeedbackThreadModel(base_models.BaseModel):
"""Threads for each exploration.
The id of instances of this class has the form
[EXPLORATION_ID].[THREAD_ID]
"""
# ID of the exploration the thread is about.
exploration_id = ndb.StringProperty(required=True, indexed=True)
# ID of state the thread is for. Does not exist if the thread is about the
# entire exploration.
state_name = ndb.StringProperty(indexed=True)
# ID of the user who started the thread. This may be None if the feedback
# was given anonymously by a learner.
original_author_id = ndb.StringProperty(indexed=True)
# Latest status of the thread.
status = ndb.StringProperty(
default=STATUS_CHOICES_OPEN,
choices=STATUS_CHOICES,
required=True,
indexed=True,
)
# Latest subject of the thread.
subject = ndb.StringProperty(indexed=False)
# Summary text of the thread.
summary = ndb.TextProperty(indexed=False)
# Specifies whether this thread has a related learner suggestion.
has_suggestion = ndb.BooleanProperty(indexed=True, default=False)
# The number of messages in the thread.
message_count = ndb.IntegerProperty(indexed=True)
# When this thread was last updated. This overrides the field in
# BaseModel. We are overriding it because we do not want the last_updated
# field to be updated everytime the feedback thread is changed. For example,
# on running the job for calculating the number of messages in a thread
# and updating the message_count field we do not wish the last_updated field
# to be updated.
last_updated = ndb.DateTimeProperty(indexed=True)
def put(self, update_last_updated_time=True):
"""Writes the given thread instance to the datastore.
Args:
update_last_updated_time: bool. Whether to update the
last_updated_field of the thread.
Returns:
thread. The thread entity.
"""
if update_last_updated_time:
self.last_updated = datetime.datetime.utcnow()
return super(FeedbackThreadModel, self).put()
@classmethod
def generate_new_thread_id(cls, exploration_id):
"""Generates a new thread ID which is unique within the exploration.
Args:
exploration_id: str. The ID of the exploration.
Returns:
str. A thread ID that is different from the IDs of all
the existing threads within the given exploration.
Raises:
Exception: There were too many collisions with existing thread IDs
when attempting to generate a new thread ID.
"""
for _ in range(_MAX_RETRIES):
thread_id = (
utils.base64_from_int(utils.get_current_time_in_millisecs()) +
utils.base64_from_int(utils.get_random_int(_RAND_RANGE)))
if not cls.get_by_exp_and_thread_id(exploration_id, thread_id):
return thread_id
raise Exception(
'New thread id generator is producing too many collisions.')
@classmethod
def generate_full_thread_id(cls, exploration_id, thread_id):
"""Generates a full thread ID, given the exploration ID and
the thread ID.
Args:
exploration_id: str. The exploration ID the thread belongs to.
thread_id: str. The thread ID.
Returns:
str. Full thread ID.
"""
return '.'.join([exploration_id, thread_id])
@classmethod
def get_exploration_and_thread_ids(cls, full_thread_ids):
"""Gives back the exploration ids and thread ids corresponding to the
full thread ids.
Args:
full_thread_ids: str. The list of full thread ids.
Returns:
list(str). The list of exploration ids to which the threads belong.
list(str). The ids of the threads corresponding to the full thread
ids.
"""
exploration_and_thread_ids = (
[thread_id.split('.') for thread_id in full_thread_ids])
return zip(*exploration_and_thread_ids)
@property
def thread_id(self):
"""Returns the thread_id (not including the exploration_id) for this
model instance.
Returns:
str. thread_id for this FeedbackThreadModel instance.
"""
return self.id.split('.')[1]
@classmethod
def create(cls, exploration_id, thread_id):
"""Creates a new FeedbackThreadModel entry.
Args:
exploration_id: str. The ID of the exploration the thread
belongs to.
thread_id: str. Thread ID of the newly-created thread.
Returns:
str. The full thread ID for the newly-created FeedbackThreadModel
instance. (This ID includes the exploration ID as a
prefix.)
Raises:
Exception: A thread with the given exploration ID
and thread ID combination exists already.
"""
instance_id = cls.generate_full_thread_id(exploration_id, thread_id)
if cls.get_by_id(instance_id):
raise Exception('Feedback thread ID conflict on create.')
return cls(id=instance_id)
@classmethod
def get_by_exp_and_thread_id(cls, exploration_id, thread_id):
"""Gets the FeedbackThreadModel entry for the given ID.
Args:
exploration_id: str. ID of the exploration.
thread_id: str. ID of the thread.
Returns:
FeedbackThreadModel|None. None if the thread is not found or is
already deleted.
"""
return cls.get_by_id(cls.generate_full_thread_id(
exploration_id, thread_id))
@classmethod
def get_threads(cls, exploration_id, limit=feconf.DEFAULT_QUERY_LIMIT):
"""Returns a list of threads associated with the exploration, ordered
by their "last updated" field. The number of entities fetched is
limited by the `limit` argument to this method, whose default
value is equal to the default query limit.
Args:
exploration_id: str.
limit: int. The maximum possible number of items
in the returned list.
Returns:
list(FeedbackThreadModel). List of threads associated with the
exploration. Doesn't include deleted entries.
"""
return cls.get_all().filter(
cls.exploration_id == exploration_id).order(
cls.last_updated).fetch(limit)
class User(ndb.Model):
crushes = ndb.IntegerProperty(repeated=True)
notifications = ndb.StringProperty(repeated=True)
signuptime = ndb.DateTimeProperty(auto_now_add=True)
interests = ndb.IntegerProperty(default=0) #0 = all, 1 = men, 2 = women
class Alert(internal_only_model.InternalOnlyModel):
"""General base class for alerts."""
# Whether the alert should only be viewable by internal users.
internal_only = ndb.BooleanProperty(indexed=True, default=False)
# The time the alert fired.
timestamp = ndb.DateTimeProperty(indexed=True, auto_now_add=True)
# Note: -1 denotes an invalid alert and -2 an ignored alert.
# By default, this is None, which denotes a non-triaged alert.
bug_id = ndb.IntegerProperty(indexed=True)
# The sheriff rotation that should handle this alert.
sheriff = ndb.KeyProperty(kind=sheriff_module.Sheriff, indexed=True)
# Each Alert is related to one Test.
test = ndb.KeyProperty(indexed=True)
# We'd like to be able to query Alerts by Master, Bot, and Benchmark names.
master_name = ndb.ComputedProperty(
lambda self: utils.TestPath(self.test).split('/')[0],
indexed=True)
bot_name = ndb.ComputedProperty(
lambda self: utils.TestPath(self.test).split('/')[1],
indexed=True)
benchmark_name = ndb.ComputedProperty(
lambda self: utils.TestPath(self.test).split('/')[2],
indexed=True)
# Each Alert has a revision range it's associated with; however,
# start_revision and end_revision could be the same.
start_revision = ndb.IntegerProperty(indexed=True)
end_revision = ndb.IntegerProperty(indexed=True)
# The group this alert belongs to.
# TODO(qyearsley): If the old AnomalyGroup entities can be removed and
# all recent groups have the kind AlertGroup, then the optional argument
# kind=alert_group.AlertGroup can be added.
group = ndb.KeyProperty(indexed=True)
# The revisions to use for display, if different than point id.
display_start = ndb.IntegerProperty(indexed=False)
display_end = ndb.IntegerProperty(indexed=False)
# Ownership data, mapping e-mails to the benchmark's owners' emails and
# component as the benchmark's Monorail component
ownership = ndb.JsonProperty()
def GetTestMetadataKey(self):
"""Get the key for the TestMetadata entity of this alert.
We are in the process of converting from Test entities to TestMetadata.
Until this is done, it's possible that an alert may store either Test
or TestMetadata in the 'test' KeyProperty. This gets the TestMetadata key
regardless of what's stored.
"""
return utils.TestMetadataKey(self.test)
@classmethod
def GetAlertsForTest(cls, test_key, limit=None):
return cls.query(cls.test.IN([
utils.TestMetadataKey(test_key),
utils.OldStyleTestKey(test_key)])).fetch(limit=limit)
class Dialeme(ndb.Model):
"""Models an individual query and response."""
question = ndb.StringProperty(indexed=False)
response = ndb.StringProperty(indexed=False)
date = ndb.DateTimeProperty(auto_now_add=True)
session_id = ndb.StringProperty(indexed=False)
class Suggestion(ndb.Model):
"""
Suggestions are generic containers for user-submitted data corrections to
the site. The generally store a model, a key, and then a json blob of
fields to append or ammend in the model.
"""
MODELS = {
"event", "match", "media", "social-media", "offseason-event",
"api_auth_access", "robot", "event_media"
}
MODEL_NAMES = {
"event": "Webcasts",
"match": "Match Videos",
"media": "Team Media",
"social-media": "Social Media",
"offseason-event": "Offseason Events",
"api_auth_access": "API Key Requests",
"robot": "CAD Models",
"event_media": "Event Videos"
}
# social-media is a Media with no year
# offseason-event is for new events (opposed to 'event' for adding webcasts to existing events)
REVIEW_ACCEPTED = 1
REVIEW_PENDING = 0
REVIEW_REJECTED = -1
REVIEW_AUTOREJECTED = -2
review_state = ndb.IntegerProperty(default=0)
reviewed_at = ndb.DateTimeProperty()
reviewer = ndb.KeyProperty(kind=Account)
author = ndb.KeyProperty(kind=Account, required=True)
contents_json = ndb.StringProperty(indexed=False) # a json blob
target_key = ndb.StringProperty() # "2012cmp"
target_model = ndb.StringProperty(choices=MODELS, required=True) # "event"
created = ndb.DateTimeProperty(auto_now_add=True, indexed=False)
updated = ndb.DateTimeProperty(auto_now=True, indexed=False)
def __init__(self, *args, **kw):
self._contents = None
super(Suggestion, self).__init__(*args, **kw)
def put(self, *args, **kwargs):
if self.review_state == 0:
user = self.author.get()
if user and user.shadow_banned:
self.review_state = self.REVIEW_AUTOREJECTED
return super(Suggestion, self).put(*args, **kwargs)
@property
def contents(self):
"""
Lazy load contents_json
"""
if self._contents is None:
self._contents = json.loads(self.contents_json)
return self._contents
@contents.setter
def contents(self, contents):
self._contents = contents
self.contents_json = json.dumps(self._contents)
@property
def candidate_media(self):
team_reference = Media.create_reference(
self.contents['reference_type'], self.contents['reference_key'])
return MediaCreator.create_media_model(self, team_reference)
@property
def youtube_video(self):
if "youtube_videos" in self.contents:
return self.contents["youtube_videos"][0]
@property
def internet_archive_video(self):
if "internet_archive_videos" in self.contents:
return self.contents["internet_archive_videos"][0]
@classmethod
def render_media_key_name(cls, year, target_model, target_key,
foreign_type, foreign_key):
"""
Keys aren't required for this model. This is only necessary if checking
for duplicate suggestions is desired.
"""
return 'media_{}_{}_{}_{}_{}'.format(year, target_model, target_key,
foreign_type, foreign_key)
@classmethod
def render_webcast_key_name(cls, event_key, webcast_dict):
"""
Keys aren't required for this model. This is only necessary if checking
for duplicate suggestions is desired.
"""
return 'webcast_{}_{}_{}_{}'.format(event_key,
webcast_dict.get('type', None),
webcast_dict.get('channel', None),
webcast_dict.get('file', None))
class Post(ndb.Model):
institute_name = ndb.StringProperty(required=True)
author = ndb.StringProperty(required=True)
publication_date = ndb.DateTimeProperty(auto_now_add=True)
likes = ndb.IntegerProperty(default=0)
