def import_to_datastore():
from models import User, Marker
from google.appengine.ext import db
my_user = User.all().filter("email", "*****@*****.**").get()
i = 0
markers = []
for data in import_data():
#old_marker = Marker.get_by_key_name(str(data["id"]))
#if old_marker:
# old_marker.delete()
marker = Marker(
key_name=str(data["id"]),
user = my_user,
title = "Accident",
description = data["description"].decode("utf8"),
address = data["address"].decode("utf8"),
location = db.GeoPt(data["lat"], data["lng"]),
type = Marker.MARKER_TYPE_ACCIDENT,
subtype = data["severity"],
created = data["date"],
modified = data["date"],
)
#marker.put()
#marker.update_location()
markers.append(marker)
print marker.key().name()
if len(markers) == 100:
print "Writing to datastore..."
db.put(markers)
markers = []
def put(self):
"""
Extends put so that it writes vaules to memcache as well as the
datastore, and keeps them in sync, even when datastore writes fails.
Returns the session object.
"""
try:
memcache.set(u"_AppEngineUtilities_Session_%s" % \
(str(self.key())), self)
except:
# new session, generate a new key, which will handle the
# put and set the memcache
db.put(self)
self.last_activity = datetime.datetime.now()
try:
self.dirty = False
db.put(self)
memcache.set(u"_AppEngineUtilities_Session_%s" % \
(str(self.key())), self)
except:
self.dirty = True
memcache.set(u"_AppEngineUtilities_Session_%s" % \
(str(self.key())), self)
return self
def post(self):
if not self.request.get('sites'):
self.error(400) # Bad request.
user = users.get_current_user()
sites = simplejson.loads(self.request.get('sites'))
# Newer clients include a 'now' parameter. If it exists, use it, otherwise,
# use the current UTC time.
day = datetime.datetime.fromtimestamp(
float(self.request.get('now', time.time()))).date()
updates = []
for (site, seconds) in sites.iteritems():
site_time = SiteTime.gql('WHERE user = :1 AND site = :2 AND day = :3',
user, site, day).get()
if not site_time:
site_time = SiteTime(user=user,
seconds=0.0,
site=site,
day=day)
site_time.seconds += seconds
updates.append(site_time)
try:
db.put(updates)
except db.TransactionFailedError:
self.error(500) # Internal server error.
user_info = UserInfo.gql('WHERE user = :1', user).get()
if not user_info:
user_info = UserInfo(user=user)
user_info.last_update=datetime.datetime.now()
user_info.put()
self.response.set_status(200)
def post(self):
import time
date = time.strftime('%Y-%m-%d', time.gmtime())
k = db.Key(self.request.get('clientkey'))
invoice = Invoices()
invoice.client = k
invoice.date = date
invoice.status = 'draft'
invoice.total = 0.00
invoice.put()
iid = invoice.key()
invoiceid = str(invoice.inum)
billedtotal = 0.00
billedtime = 0.00
# there is a form that lists the projects, allowing the user to
# check one or more the field data is the key for that project
# get_all gives an array of keys returned which we want to pull
# time from
projects = self.request.get_all('projects')
# start looping through the keys
for projectkey in projects:
# make the string key an actual key object
pkey = db.Key(projectkey)
# get everything out of the time store with that project
# associated and which has a status of logged.
times_query = Time.all()
times_query.filter('project =', pkey)
times_query.filter('status =', 'logged')
times = times_query.fetch(100)
for time in times:
time.invoice = iid
time.status = "invoiced"
db.put(time)
billedtime = float(billedtime) + float(time.hours)
billedtotal = float(billedtotal) + float(time.total)
project_update = db.get(pkey)
project_update.status = "empty"
project_update.put()
totalhoursbilled = "%2.f" % billedtime
totalbill = "%.2f" % billedtotal
totalbill = float(totalbill)
invoice_update = db.get(iid)
invoice_update.totalhours = float(totalhoursbilled)
invoice_update.totalbill = totalbill
invoice_update.put()
project_update = db.get(pkey)
project_update.billed = billedtime
project_update.put()
action = '/invoice?iid=' + str(iid)
self.redirect(action)
def _updateField(self, entity, entity_properties, name):
"""Called when the fields of the mentor are updated
When status is changed to invalid, removes the Mentor from all Student
Proposals possible mentor lists.
"""
from soc.logic.models.student_proposal import logic \
as student_proposal_logic
value = entity_properties[name]
if name == 'status' and value != entity.status and value == 'invalid':
fields = {'org': entity.scope}
# TODO make this work for more then 1000 entities
proposals_query = student_proposal_logic.getQueryForFields(fields)
# store all updated proposals
changed = []
for proposal in proposals_query:
if proposal.possible_mentors.count(entity.key()):
# remove from list and add to changed
proposal.possible_mentors.remove(entity.key())
changed.append(proposal)
# store all changed proposals
db.put(changed)
return super(Logic, self)._updateField(entity, entity_properties, name)
def get(self):
category_key = self.request.get('id')
category = db.get(category_key)
category.reverse = True
db.put(category)
memcache.delete(category.getReviewPairsKey())
self.redirect('category?id=' + category_key)
def post(self):
if ( self.request.get("img1") ):
PresentTFrame = TblFrame.all()
PresentTFrame.order('FrameNum').fetch(limit=1)
NextId = 0
for r in PresentTFrame:
if PresentTFrame:
NextId = r.FrameNum + 1
else:
NextId = 0
TFrame = TblFrame()
TFrame.FrameNum = NextId
Img = images.resize(self.request.get("img1"), 600, 600)
TFrame.FrameImage = db.Blob(Img)
TFrame.FrameDate = datetime.datetime.today() + datetime.timedelta(hours=9)
TFrame.FrameIPAddress = os.environ['REMOTE_ADDR']
db.put(TFrame)
cUrl = "/FrameForm"
self.redirect( cUrl )
else:
self.redirect( '/Err' )
def post(self):
template_values = {}
user_data = UserData.current()
status_file = StringIO.StringIO(self.request_string("status_file"))
reader = csv.reader(status_file)
student_list = []
for line in reader:
student_email = line[0]
student_status = line[1]
student_comment = line[2]
student = SummerStudent.all().filter("email =", student_email).get()
if student is None:
logging.error("Student %s not found" % student_email)
continue
student.application_status = student_status
student.comment = student_comment
if student_status == "Accepted":
student.accepted = True
student_list.append(student)
db.put(student_list)
self.response.out.write("OK")
self.response.set_status(200)
def set(self, quiz_taker, save=False):
logging.info('setting proficiency level data for %s', quiz_taker.unique_identifier)
save = []
pro_scores = {}
for tl in quiz_taker.topic_levels.fetch(1000):
try:
current_average = pro_scores[tl.topic.proficiency.key()]['average']
current_count = pro_scores[tl.topic.proficiency.key()]['count']
current_product = current_average * current_count
new_sum = current_product + tl.topic_level
new_count = current_count + 1
new_average = new_sum / new_count
except: # no current average
pro_scores[tl.topic.proficiency.key()] = {}
new_count = 1
new_average = tl.topic_level
pro_scores[tl.topic.proficiency.key()]['count'] = new_count
pro_scores[tl.topic.proficiency.key()]['average'] = new_average
for pro_pair in pro_scores.items():
pl_keyname = str(quiz_taker.unique_identifier) + "_" + str(pro_pair[0])
pro_pair[1]['average'] = int(pro_pair[1]['average'])
if ProficiencyLevel.get_by_key_name(pl_keyname):
proficiency_level = ProficiencyLevel.get_by_key_name(pl_keyname)
proficiency_level.proficiency_level = pro_pair[1]['average']
else:
proficiency_level = ProficiencyLevel(key_name = pl_keyname,
proficiency = pro_pair[0],
quiz_taker = quiz_taker,
proficiency_level = pro_pair[1]['average'])
save.append(proficiency_level)
if save: db.put(save)
return save
def upload_model(self,model_class,result):
from google.appengine.ext import db
from utils import deserialize
save = []
to_delete = []
for entity in result:
if "id" in entity:
existing_entry = model_class.get(db.Key.from_path(model_class.kind(),entity ["id"]))
if existing_entry:
to_delete.append(existing_entry) # Remove the existing entry with numeric ID
object = deserialize(model_class , entity)
save.append(object)
if len(to_delete) > 100:
db.delete(to_delete)
to_delete = []
if len(save) > 100:
db.put(save)
save = []
db.delete(to_delete)
db.put(save)
def put(self):
"""
Extends put so that it writes vaules to memcache as well as the
datastore, and keeps them in sync, even when datastore writes fails.
It also uses a db.put(), rather than the ROTModel put, to avoid
retries on puts. With the memcache layer this optimizes performance,
stopping on db.Timeout rather than retrying.
Returns the session object.
"""
if self.session_key:
memcache.set(u"_AppEngineUtilities_Session_%s" % \
(unicode(self.session_key)), self)
else:
# new session, generate a new key, which will handle the
# put and set the memcache
self.create_key()
self.last_activity = datetime.datetime.now()
try:
self.dirty = False
db.put(self)
memcache.set(u"_AppEngineUtilities_Session_%s" % \
(unicode(self.session_key)), self)
except:
self.dirty = True
memcache.set(u"_AppEngineUtilities_Session_%s" % \
(unicode(self.session_key)), self)
return self
def post(self):
session_user = self.get_session_user()
keys = json.loads(self.request.get('keys'))
user = models.User.get_user_from_identifier(session_user.identifier)
orders = []
books = []
isbn_list = []
for key in keys:
book = db.get(db.Key(key))
order = models.UserBookOrder(user=user, book=book)
books.append(str(book))
orders.append(order)
db.put(orders)
person = user.people_singleton[0]
book_infos = '\n'.join(books)
queue_mail_task(url='/worker/mail/book_order/',
params=dict(
user_key=str(user.key()),
books=book_infos,
person_name=str(person),
person_email_address=user.email,
person_phone_number = str(person.phones[0])
),
method='POST'
)
self.response.out.write(json.dumps(keys))
def post(self):
""" this handler supports http post """
google_user = google_users.get_current_user()
user = db.Query(Users).filter('user_id =', google_user.user_id()).get()
if user is None:
user = Users(user_id = google_user.user_id(), email = google_user.email())
emails = self.request.get('email').lower()
emails = emails.split(',')
if len(emails) == 0:
self.response.set_status(400)
self.response.out.write('Email is required.')
return
added_one = False
for email in emails:
if email.strip() == '':
continue
try:
user.pt_emails.index(email)
except ValueError:
added_one = True
user.pt_emails.append(email)
if added_one == False:
self.response.set_status(400)
self.response.out.write('Email is required.')
return
db.put(user)
self.response.out.write(json.dumps(user.pt_emails))
def create_namespace_entry(self, namespace, size, number, timestamp):
""" Puts a namespace into the datastore.
Args:
namespace: A string, the namespace.
size: An int representing the number of bytes taken by a namespace.
number: The total number of entities in a namespace.
timestamp: A datetime.datetime object.
Returns:
True on success, False otherwise.
"""
entities_to_write = []
namespace_stat = stats.NamespaceStat(subject_namespace=namespace,
bytes=size,
count=number,
timestamp=timestamp)
entities_to_write.append(namespace_stat)
# All application are assumed to have the default namespace.
if namespace != "":
namespace_entry = metadata.Namespace(key_name=namespace)
entities_to_write.append(namespace_entry)
try:
db.put(entities_to_write)
except datastore_errors.InternalError, internal_error:
logging.error("Error inserting namespace info: {0}.".\
format(internal_error))
return False
def _put(models,countdown=0):
batch_size = 50
to_put = []
keys = []
try:
last_index = 0
for i,model in enumerate(models):
to_put.append(model)
last_index = i
if (i+1) % batch_size == 0:
keys.extend(db.put(to_put))
to_put = []
keys.extend(db.put(to_put))
return keys
except apiproxy_errors.DeadlineExceededError:
keys.extend(db.put(to_put))
deferred.defer(_put,models[last_index+1:],_countdown=10)
return keys
except apiproxy_errors.CapabilityDisabledError:
if not countdown:
countdown = 30
else:
countdown *= 2
deferred.defer(_put,models,countdown,_countdown=countdown)
def get(self,time):
ticker = "yhoo"
ticker_instance = Stock.get_by_key_name(ticker)
# time = how many hours back in the past we go
# loop between now-time and now-time-1
start_time = datetime.now() - timedelta(hours=int(time))
end_time = start_time - timedelta(hours=1)
loop_time = start_time
# Check we aren't already covering this time period
exists_query = TweetBag.all().filter("stock = ", ticker_instance).filter("time >", end_time).filter("time <", start_time).order("-time")
if exists_query.count() > 0:
highest = exists_query.get()
end_time = highest.time
inserts = []
while loop_time > end_time:
num_tweets = 0
loop_time_5 = loop_time - timedelta(minutes=5)
for keyword in ticker_instance.keywords:
tweet_query = Tweet.all().filter("keywords = ", keyword).filter("created_at > ", loop_time_5).filter("created_at < ", loop_time)
num_tweets += tweet_query.count()
bagobj = TweetBag (
sample_tweet = tweet_query.get(),
stock = ticker_instance,
count = num_tweets,
time = loop_time_5
)
inserts.append(bagobj)
loop_time = loop_time_5
db.put(inserts)
def do_task_update_users(force, cursor):
force = bool(int(force))
cursor = int(cursor)
friend_list, data = api._GetFriends(cursor=cursor)
batch = []
for friend in friend_list:
username = friend.GetScreenName()
isnew = not CurryUser.get_by_key_name(username)
if force or isnew:
batch.append(CurryUser(key_name=username))
taskqueue.add(
url=('/task/update_links/%s/%d' % (username, (isnew and not force))),
queue_name='update-links-queue'
)
db.put(batch)
logging.debug('added %d users from %d friends of bot' % (len(batch), len(friend_list)))
logging.debug('next cursor=%d' % data['next_cursor'])
if int(data['next_cursor']) != 0:
taskqueue.add(
url=('/task/update_users/%d/%d' % (force, int(data['next_cursor']))),
queue_name='update-users-queue'
)
else:
logging.info('update users: done')
return 'ok'
def load(self):
save = []
logging.info('loading fixture')
self.fixture_offset = Setting.get_by_key_name('fixture_offset')
if self.fixture_offset.status == "update_stats": #instead of 'create_account'
logging.warning('Load Fixtures Cron Job Hit Twice In a Row')
print "error -- current status: ", self.fixture_offset.status
return False
this_account, this_user, this_quiz_taker = self.get_fixture()
scores = Scores()
import random
correct_prob = random.randint(80,95)
FIXTURE_PROFICIENCY = self.get_fixture_subject()
from model.proficiency import Proficiency
this_proficiency = random.sample( Proficiency.gql("WHERE status = 'public'").fetch(1000), 1 )[0]
save_scores = scores.make_scores(this_user, this_proficiency, correct_prob, SCORE_NUM = 10)
memcache.set('current_fixture', ( this_account, this_user, this_quiz_taker ), 600000)
self.fixture_offset.status = "update_stats"
print this_user.nickname
save.append(self.fixture_offset)
save.extend(save_scores)
db.put(save)
# open fixture.xml, go to offset.
# load one name, email pair. register.
FIXTURE_PROFICIENCY = self.get_fixture_subject()
def update_stats(self):
save = []
self.fixture_offset = Setting.get_by_key_name('fixture_offset')
if self.fixture_offset.status != "update_stats":
logging.warning('Update Stats Job Hit Twice In A Row')
print "error -- current status: ", self.fixture_offset.status
return False
(this_account, this_user, this_quiz_taker) = memcache.get('current_fixture')
logging.info('Updating User Stats for User %s', this_user.unique_identifier)
from quiztaker.methods import ProficiencyLevels
pl = ProficiencyLevels()
pl.set_for_user(this_quiz_taker)
from accounts.methods import Awards
awards = Awards()
# check for new awards
new_awards = awards.check_all(this_quiz_taker)
from accounts.methods import Sponsorships
sponsorships = Sponsorships()
# check for new sponsorships, both personal and business
new_sponsorships = sponsorships.check_user(this_user)
self.fixture_offset.value += 1
self.fixture_offset.status = "create_account"
save.append(self.fixture_offset)
db.put(self.fixture_offset)
return new_awards, new_sponsorships
def bind(domain_string, mydomain):
"""
Calculate and save in db the last assignment valid to short classic
Params
domain_s - a domain : string
mydomain - a domain who provide the service : string
Return
last assignament : string
"""
last_assignment = ''
query = db.Query(Domain).filter('name_domain = ', domain_string)
if query.count() != 0:
dt = query.fetch(1)[0]
last_assignment = dt.last_assignament
while True:
last_assignament = compute_next(last_assignment)
short = '/'.join( (mydomain, domain_string, last_assignament) )
if not functions.isAlredyCustum(short):
break
dt.last_assignament = last_assignament
db.put(dt)
return dt.last_assignament
def GetSnippetUser(user=None):
"""Try to get the Snipper user from the datastore, create if not.
Args:
user: User object.
Returns:
SnippetUser object.
"""
if user is None:
user = users.get_current_user()
logging.debug('GetSnippetUser call for %s', user)
logging.debug('Trying memcache first...')
memcache_key = 'SnippetUser-' + str(user)
snippet_user = memcache.get(memcache_key)
# Added a check for the timezone to ensure the SnipperUser version is updated.
if snippet_user and snippet_user.timezone:
logging.debug('Memcache worked, returning.')
return snippet_user
else:
snippet_user = db.Query(SnippetUser).filter('User = '******'Adding new Snipper user: %s', user)
timezone = util.GetUserTimezone(user.nickname())
snippet_user = SnippetUser()
snippet_user.User = user # pylint: disable-msg=C6409
if timezone:
snippet_user.timezone = timezone
db.put(snippet_user)
memcache.set(memcache_key, snippet_user)
return snippet_user
def put_expiry_flags(self):
"""Updates the is_expired flags on this Person and related Notes to
make them consistent with the effective_expiry_date() on this Person,
and commits the changes to the datastore."""
import utils
now = utils.get_utcnow()
expired = self.get_effective_expiry_date() <= now
if self.is_expired != expired:
# NOTE: This should be the ONLY code that modifies is_expired.
self.is_expired = expired
# if we neglected to capture the original_creation_date,
# make a best effort to grab it now, for posterity.
if not self.original_creation_date:
self.original_creation_date = self.source_date
# If the record is expiring (being replaced with a placeholder,
# see http://zesty.ca/pfif/1.3/#data-expiry) or un-expiring (being
# restored from deletion), we want the source_date and entry_date
# updated so downstream clients will see this as the newest state.
self.source_date = now
self.entry_date = now
# All the Notes on the Person also expire or unexpire, to match.
notes = self.get_notes(filter_expired=False)
for note in notes:
note.is_expired = expired
# Store these changes in the datastore.
db.put(notes + [self])
def test_addOfficeHour(self):
b = F.addBuilding("Office Hour Test", "ET")
l = F.addLocation(b, "2nd", "201")
s = F.addSchedule("1/1/2000", "1pm", ["Monday", "Tuesday"], "2pm", "1/1/2010")
oh = F.addOfficeHour(s, l)
self.assert_(oh.type == "Office Hours")
self.assert_(oh.location.building.name == "Office Hour Test" and oh.location.building.abbreviation == "ET")
self.assert_(oh.location.floor == "2nd" and oh.location.room == "201")
self.assert_(oh.schedule.start_date == "1/1/2000" and oh.schedule.end_date == "1/1/2010")
self.assert_(oh.schedule.start_time == "1pm" and oh.schedule.end_time == "2pm")
self.assert_("Monday" in oh.schedule.days and "Tuesday" in oh.schedule.days)
db.put(oh)
key = oh.key()
oh = db.get(key)
self.assert_(oh.type == "Office Hours")
self.assert_(oh.location.building.name == "Office Hour Test" and oh.location.building.abbreviation == "ET")
self.assert_(oh.location.floor == "2nd" and oh.location.room == "201")
self.assert_(oh.schedule.start_date == "1/1/2000" and oh.schedule.end_date == "1/1/2010")
self.assert_(oh.schedule.start_time == "1pm" and oh.schedule.end_time == "2pm")
self.assert_("Monday" in oh.schedule.days and "Tuesday" in oh.schedule.days)
def post(self):
user = users.get_current_user()
if not user:
self.response.set_status(401)
return
text = self.request.get("text")
line = ChatLine(user=user, text=text)
db.put(line)
# Send an update to everyone who's listening.
# TODO(danvk): write a function to do this.
msg = { 'lines': [ "%s: %s" % (user.nickname(), text) ] }
active_users=db.GqlQuery("SELECT * FROM ActiveUsers")
for user in active_users:
if (datetime.datetime.now() - user.last_update_time
> datetime.timedelta(hours=1)):
logging.info("Removing inactive user: "******"Sending message on channel: " + user.user.user_id())
try:
channel.send_message(user.user.user_id(), simplejson.dumps(msg))
except channel.InvalidChannelKeyError:
# This happens when you restart the server and sever connections.
pass
self.response.out.write('ok')
def test_addRetrieveWebsite(self):
ra = F.addResearchGroup("Beer drinkers anonymous")
db.put(ra)
auth1 = F.addName("Author", "One")
auth2 = F.addName("Author", "Two")
auth1_key = db.put(auth1)
auth2_key = db.put(auth2)
auths = [auth1, auth2]
w = F.addWebsite("http://beer.com", ra, auths)
self.assert_(w.address == "http://beer.com")
self.assert_(w.group.name == "Beer drinkers anonymous")
# self.assert_(len(w.authors) == 2)
self.assert_(auth1_key in w.authors and auth2_key in w.authors)
w_key = db.put(w)
w = db.get(w_key)
self.assert_(w.address == "http://beer.com")
self.assert_(w.group.name == "Beer drinkers anonymous")
# self.assert_(len(w.authors) == 2)
self.assert_(auth1_key in w.authors and auth2_key in w.authors)
def _set_css_deferred(user_data_key, video_key, status, version):
user_data = user_models.UserData.get(user_data_key)
uvc = UserVideoCss.get_for_user_data(user_data)
css = pickle_util.load(uvc.pickled_dict)
id = '.v%d' % video_key.id()
if status == UserVideoCss.STARTED:
if id in css['completed']:
logging.warn("video [%s] for [%s] went from completed->started. ignoring." %
(video_key, user_data_key))
else:
css['started'].add(id)
else:
css['started'].discard(id)
css['completed'].add(id)
uvc.pickled_dict = pickle_util.dump(css)
uvc.load_pickled()
# if set_css_deferred runs out of order then we bump the version number
# to break the cache
if version < uvc.version:
version = uvc.version + 1
user_data.uservideocss_version += 1
db.put(user_data)
uvc.version = version
db.put(uvc)
def CreateMachines(num_instances, token, os, browser, browser_version, download_info, retries=0):
"""Create and launch EC2 machines for the given parameters.
Args:
num_instances: An integer representing the number of instances to spawn
with the given configuration.
token: A string representing the token for this run.
os: An integer that corresponds to an enum.OS value.
browser: An integer that corresponds to an enum.BROWSER value.
browser_version: A string representing browser version to use.
Specifically, this should be the channel for Chrome.
download_info: A string representing the information necessary for
calculating the version and browser download url for the given machine.
retries: An optional paramater specifying the initial retry count for the
machine.
"""
logging.info(
"\n".join(["num_instances: %d", "token: %s", "os: %d", "browser: %d", "browser_version: %s"]),
num_instances,
token,
os,
browser,
browser_version,
)
ec2 = ec2_manager.EC2Manager()
user_data = simplejson.dumps(
{"channel": browser_version, "os": OS_TO_USER_DATA[os], "token": token, "download_info": download_info}
)
logging.info("Spawning EC2 machines.")
# Note: All exceptions are caught here because the EC2 API could fail after
# successfully starting a machine. Because this task is rescheduled on
# failure, we need to make sure we don't keep spawning EC2 machines.
try:
bots_instances = ec2.StartAmiWithOs(
os, count=num_instances, instance_type=DEFAULT_INSTANCE_SIZE, user_data=user_data
)
except Exception:
logging.exception("Something failed when setting up the EC2 instance. " "Stopping setup for this instance.")
return
logging.info("Creating the corresponding ClientMachine models.")
new_instances = []
for instance in bots_instances:
new_instances.append(
client_machine.ClientMachine(
vm_service=enum.VM_SERVICE.EC2,
os=os,
browser=browser,
browser_version=browser_version,
client_id=instance.inst_id,
status=enum.MACHINE_STATUS.PROVISIONED,
retry_count=retries,
token=token,
download_info=download_info,
)
)
db.put(new_instances)
logging.info("Finished creating the ClientMachine models.")
def test_addRetrieveArticle(self):
article = F.addArticle("Psychology in the Lab", "47000")
self.assert_(article.journal == "Psychology in the Lab" and article.edition == "47000")
db.put(article)
key = article.key()
article = db.get(key)
self.assert_(article.journal == "Psychology in the Lab" and article.edition == "47000")
def append_api_response(self, checkins_json_data):
seen_venue_ids = set()
new_venues = []
new_count = 0
for checkin_json_data in checkins_json_data['checkins']['items']:
checkin = data.checkin.Checkin(checkin_json_data)
if 'venue' in checkin_json_data:
venue_json_data = checkin_json_data['venue']
if 'id' in venue_json_data:
venue_id = venue_json_data['id']
if venue_id not in seen_venue_ids:
venue, is_new, = data.venue.Venue.create_if_needed(venue_json_data)
seen_venue_ids.add(venue_id)
if is_new:
new_venues.append(venue)
if checkin.id not in self._checkins_by_id:
new_count += 1
self._checkins_by_id[checkin.id] = checkin
if new_venues:
logging.info('Saving %d new venues', len(new_venues))
db.put(new_venues)
return new_count
def _new_chat(self):
'''Attempts to find a good-matching open chat, if none are found it returns a new open chat with the current
user as host.
'''
user = users.get_current_user()
metrics = db.Query(UserMetrics).filter('user = '******'guest = ', None).filter('nsfw = ', False)
vocab = set(metrics.twl)
matches = []
for chat in open_chats:
host = db.Query(UserMetrics).filter('user =', chat.host).get()
chat.score = len(vocab.intersection(host.twl))*10 - abs(host.awl-metrics.awl) - abs(host.act-metrics.act)
if chat.score > -1:
matches.append(chat)
if matches:
matches.sort(key = lambda chat: chat.score, reverse=True)
for chat in matches:
result = db.run_in_transaction(join_chat(chat.key(), user))
if result is not None:
return result
chat = Chat()
chat.created = datetime.now()
chat.host = user
chat.nsfw = False
chat.finished = False
chat.guest = None
db.put(chat)
return chat
def import_schedule(season=None):
# We want to convert from Eastern Time in the schedule to UTC
import pytz
est = pytz.timezone('US/Eastern')
if not season:
season = Season.current()
# Make a map of all the teams. Map by both name and place because the
# input data uses either one, sort of arbitrarily.
teams = Team.all().fetch(32)
team_map = dict((t.name, t) for t in teams)
team_map.update(dict((t.place, t) for t in teams))
# We'll build up the weeks and games we're importing in these structures.
week_map = {}
games = []
csvlines = open('data/examples/2011-2012-schedule.csv').readlines()[1:]
reader = csv.reader(csvlines)
for row in reader:
# Figure out kickoff
kickoff = '%s %s' % itemgetter(0, 7)(row)
kickoff = datetime.datetime.strptime(kickoff, '%m/%d/%Y %I:%M %p')
kickoff = est.localize(kickoff).astimezone(pytz.utc)
# Look up home and away teams by team name in the team map
team_names = [_.strip() for _ in row[-2:]]
home, away = itemgetter(*team_names)(team_map)
# Figure out what week this game belongs to. The data in the CSV is a
# string like this:
# 'NFL Week 8: Chargers @ Chiefs [TV: ESPN]'
info = row[2]
week_num = int(info.split()[2][:-1])
if week_num not in week_map:
key = db.Key.from_path('Week', week_num, parent=season.key())
week_map[week_num] = Week(key=key, name='Week %s' % week_num)
week = week_map[week_num]
key_name = '%s@%s' % (away.slug, home.slug)
game = dict(parent=week,
key_name=key_name,
home_team=home,
away_team=away,
teams=[home.key(), away.key()],
start=kickoff)
games.append(game)
games.sort(key=itemgetter('parent'))
# Figure out each week's end date based on the latest kickoff of its games
for week, week_games in groupby(games, itemgetter('parent')):
week_games = list(week_games)
start = min(game['start'] for game in week_games)
end = max(game['start'] for game in week_games)
week.start = start
week.end = end + datetime.timedelta(hours=5)
# Store the weeks, so they have "complete" keys, and can therefore be used
# as the parents to the games we're about to create.
week_keys = db.put(week_map.values())
# Create actual game entities from the kwargs we gathered, and store them.
games = [Game(**kwargs) for kwargs in games]
game_keys = db.put(games)
return week_keys + game_keys
def get_ads(cursor=None, count=0):
print "getting ads %d" % count
ads = Ad.all().filter("source =", "avito").order("-created_at")
print ads[0].created_at
if cursor:
ads = ads.with_cursor(cursor)
ads_for_put = []
ads_for_delete = []
for ad in ads.fetch(10):
try:
parser = adsparser.parse(ad.key().name(), 'spb')
except StandardError as e:
msg = e.__str__()
if msg == 'HTTP Error 404: Not found':
print "deleting"
ad.deleted = True
ads_for_put.append(ad)
continue
if parser.phone_key is None:
continue
phone_url = "%s?pkey=%s" % (ad.key().name().replace(
'items/', 'items/phone/'), parser.phone_key)
phone_cmd = command.replace("__url__", phone_url)
print ad.key().name()
fin, fout = os.popen4(phone_cmd)
phone = fout.read()
time.sleep(2)
f = open("result.txt", "r")
phone = adsparser.format_phone(f.read())
f.close()
if parser.is_real_agent:
ad.rating = 0
else:
if ad.phone is None or ad.phone == '':
ad.rating = 100
if ad.phone is not None and ad.phone != '' and ad.phone != phone:
new_ad = clone_entity(ad,
key_name="%s?v2" % ad.key().name(),
parent=ad)
new_ad.phone = phone
new_ad.created_at = datetime.datetime.now()
ads_for_put.append(new_ad)
if ad.phone is None or ad.phone == '':
ad.phone = phone
ad.created_at = datetime.datetime.combine(
ad.created_at.date(),
datetime.datetime.now().time())
ads_for_put.append(ad)
print "saving ads"
db.put(ads_for_put)
print "ads saved"
for ad in ads_for_put:
try:
print "adding task"
taskqueue.add(queue_name='quick',
url='/ad/check',
params={'key': ad.key().name()})
except:
pass
print "tasks added"
get_ads(ads.cursor(), count + 10)
def confirm_note_with_bad_words(self, note):
"""After a note containing bad words is confirmed by the author,
we will:
(1) set note.confirmed = True;
(2) copy the note from NoteWithBadWords to Note;
(3) log user action;
(4) update person record. """
note.confirmed = True
# Check whether the record author disabled notes on
# this record during the time between the note author inputs the
# note in the UI and confirms the note through email.
person = model.Person.get(self.repo, note.person_record_id)
if person.notes_disabled:
return self.error(
200, _('The author has disabled notes on this record.'))
# Check whether the admin disabled reporting "believed_dead"
# during the time between the note author inputs the
# note in the UI and confirms the note through email.
if (self.params.status == 'believed_dead'
and not self.config.allow_believed_dead_via_ui):
return self.error(
200,
_('Not authorized to post notes with the status '
'"believed_dead".'))
# clone the flagged note to Note table.
note_confirmed = model.Note.create_original(
self.repo,
entry_date=note.entry_date,
person_record_id=note.person_record_id,
author_name=note.author_name,
author_email=note.author_email,
author_phone=note.author_phone,
source_date=note.source_date,
author_made_contact=note.author_made_contact,
status=note.status,
email_of_found_person=note.email_of_found_person,
phone_of_found_person=note.phone_of_found_person,
last_known_location=note.last_known_location,
text=note.text)
entities_to_put = [note_confirmed]
note.confirmed_copy_id = note_confirmed.get_record_id()
entities_to_put.append(note)
# Specially log 'believed_dead'.
if note_confirmed.status == 'believed_dead':
model.UserActionLog.put_new('mark_dead', note_confirmed,
person.primary_full_name,
self.request.remote_addr)
# Specially log a switch to an alive status.
if (note_confirmed.status in ['believed_alive', 'is_note_author']
and person.latest_status
not in ['believed_alive', 'is_note_author']):
model.UserActionLog.put_new('mark_alive', note_confirmed,
person.primary_full_name)
# Update the Person based on the Note.
if person:
person.update_from_note(note_confirmed)
# Send notification to all people
# who subscribed to updates on this person
subscribe.send_notifications(self, person, [note_confirmed])
entities_to_put.append(person)
# Write one or both entities to the store.
db.put(entities_to_put)
def store(): db.put(doc)
def make_doc():
db.put(doc)
for scan in scans:
scan.lacks_document = False
scan.document = doc.key()
db.put(scan)
def post(self):
body = self.request.body.decode('utf-8').encode(
'ascii', 'xmlcharrefreplace')
logging.info('Post body is %d characters', len(body))
topic = get_self_link(self.request)
data = feedparser.parse(body)
if data.bozo:
logging.error('Bozo feed data. %s: %r',
data.bozo_exception.__class__.__name__,
data.bozo_exception)
if (hasattr(data.bozo_exception, 'getLineNumber')
and hasattr(data.bozo_exception, 'getMessage')):
line = data.bozo_exception.getLineNumber()
logging.error('Line %d: %s', line,
data.bozo_exception.getMessage())
segment = body.split('\n')[line - 1]
logging.info('Body segment with error: %r', segment)
return self.response.set_status(500)
update_list = []
logging.info('Found %d entries', len(data.entries))
for entry in data.entries:
if hasattr(entry, 'content'):
# This is Atom.
entry_id = entry.id
content = entry.content[0].value
link = entry.get('link', '')
title = entry.get('title', '')
else:
content = entry.get('description', '')
title = entry.get('title', '')
link = entry.get('link', '')
entry_id = (entry.get('id', '') or link or title or content)
logging.info(
'Found entry in topic = "%s" with title = "%s", id = "%s", '
'link = "%s", content = "%s"', topic, title, entry_id, link,
content)
update_list.append(
TopicUpdate(key_name='key_' +
hashlib.sha1(link + '\n' + entry_id).hexdigest(),
topic=topic,
title=title,
content=content,
link=link,
callback=self.request.path[len('/subscriber'):]))
db.put(update_list)
self.response.set_status(200)
self.response.out.write('Aight. Saved.')
uploads = len(update_list)
uploads_not_claimed = 0
for entry in update_list:
yt_video_id = get_yt_video_id(entry.link)
if (yt_video_id):
try:
apply_usage_policy(yt_video_id)
except:
uploads_not_claimed = uploads_not_claimed + 1
logging.info("Videos claimed: %d of %d", uploads - uploads_not_claimed,
uploads)
def post(self):
"""Save the user's preferences."""
user = users.get_current_user()
snipper_user = models.GetSnippetUser(user)
logging.debug('Saving settings for %s', user)
errors = []
date_format = str(
self.request.get('date_format', snipper_user.date_format))
snippet_format = self.request.get('snippet_format',
snipper_user.snippet_format)
snipper_user.mail_snippets = bool(
self.request.get('mail_snippets', False))
snipper_user.send_confirm = bool(
self.request.get('send_confirm', False))
snipper_user.reset_day = int(
self.request.get('reset_day', snipper_user.reset_day))
snipper_user.reset_hour = int(
self.request.get('reset_hour', snipper_user.reset_hour))
timezone = self.request.get('timezone')
try:
assert pytz.timezone(timezone)
except pytz.UnknownTimeZoneError:
logging.exception('Invalid timezone: %s', timezone)
errors.append('Invalid timezone: %s.' % timezone)
else:
snipper_user.timezone = timezone
# Convert to UTC for storage.
utc_reset = util.ResetDatetimeToUtc(snipper_user.reset_day,
snipper_user.reset_hour,
snipper_user.timezone)
snipper_user.utc_reset_day = utc_reset.weekday()
snipper_user.utc_reset_hour = utc_reset.hour
try:
assert datetime.datetime.now().strftime(date_format)
except (ValueError, TypeError):
errors.append('Invalid date format "%s".' % date_format)
logging.exception('date_format "%s" failed validation.',
date_format)
else:
snipper_user.date_format = date_format
try:
assert snippet_format % 'test snippet'
except (ValueError, TypeError):
errors.append('Invalid snippet format "%s".' % snippet_format)
logging.exception('snippet_format "%s" is invalid.',
snippet_format)
else:
snipper_user.snippet_format = snippet_format
logging.debug(
'date:%s, snip:%s, mail:%s, conf:%s, day:%s, hour:%s, tz:%s'
'utc_day:%s, utc_hour:%s', snipper_user.date_format,
snipper_user.snippet_format, snipper_user.mail_snippets,
snipper_user.send_confirm, snipper_user.reset_day,
snipper_user.reset_hour, snipper_user.timezone,
snipper_user.utc_reset_day, snipper_user.utc_reset_hour)
try:
db.put(snipper_user)
except (db.Timeout, db.InternalError):
logging.exception('Could not save settings.')
errors.append('Could not save settings.')
else:
memcache_key = 'SnippetUser-' + str(user)
memcache.set(memcache_key, snipper_user)
if errors:
errors = urllib.quote_plus(','.join(errors))
return self.redirect('/settings?errors=' + errors)
# Drop the last two weeks from memcache.
memcache.delete_multi(
['snippets_%s_%d' % (str(user), x) for x in (0, 1)])
self.redirect('/?msg=Settings+saved.')
def setUp(self):
self.testbed = testbed.Testbed()
self.testbed.activate()
self.testbed.init_user_stub()
self.testbed.init_datastore_v3_stub()
self.testbed.init_memcache_stub()
self.testbed.init_taskqueue_stub()
model.Repo(key_name='haiti').put()
logging.basicConfig(level=logging.INFO, stream=sys.stderr)
self.mox = None
# Setup cheerfully stolen from test_model.
set_utcnow_for_test(datetime.datetime(2010, 1, 1))
self.photo = model.Photo.create('haiti', image_data='xyz')
self.photo.put()
self.photo_key = self.photo.key()
self.p1 = model.Person.create_original(
'haiti',
given_name='John',
family_name='Smith',
home_street='Washington St.',
home_city='Los Angeles',
home_state='California',
home_postal_code='11111',
home_neighborhood='Good Neighborhood',
author_name='Alice Smith',
author_phone='111-111-1111',
author_email='*****@*****.**',
photo_url='',
photo=self.photo,
source_url='https://www.source.com',
source_date=datetime.datetime(2010, 1, 1),
source_name='Source Name',
entry_date=datetime.datetime(2010, 1, 1),
expiry_date=datetime.datetime(2010, 2, 1),
other='')
self.p2 = model.Person.create_original(
'haiti',
given_name='Tzvika',
family_name='Hartman',
home_street='Herzl St.',
home_city='Tel Aviv',
home_state='Israel',
source_date=datetime.datetime(2010, 1, 1),
entry_date=datetime.datetime(2010, 1, 1),
expiry_date=datetime.datetime(2010, 3, 1),
other='')
self.key_p1 = db.put(self.p1)
self.key_p2 = db.put(self.p2)
self.n1_1 = model.Note.create_original(
'haiti',
person_record_id=self.p1.record_id,
linked_person_record_id=self.p2.record_id,
status=u'believed_missing',
author_made_contact=False,
entry_date=get_utcnow(),
source_date=datetime.datetime(2010, 1, 2))
self.note_id = self.n1_1.note_record_id
db.put(self.n1_1)
self.to_delete = [self.p1, self.p2, self.n1_1, self.photo]
def get(self, action=None):
if action:
if action == 'turn_download_on':
turn_download_on()
if action == 'turn_download_off':
turn_download_off()
self.response.out.write('Admin page<br/><br/>')
self.response.out.write('<a href="/gt/">Home</a><br/><br/>')
self.response.out.write(
'<a href="/gt/admin/create_geo_trees">Create GeoTrees</a><br/><br/>'
)
self.response.out.write(
'<a href="/_ah/admin">App Engine localhost admin</a><br/><br/>')
self.response.out.write(
'<a href="/gt/admin/add_points">Add OSM points to GeoTree</a><br/>'
)
self.response.out.write(
'<a href="/gt/admin/add_cities">Add cities to GeoTree</a><br/><br/>'
)
self.response.out.write(
'<a href="/gt/admin/update_tiles">Update OSM GeoTree tiles</a><br/>'
)
self.response.out.write(
'<a href="/gt/admin/update_cities_tiles">Update Cities GeoTree tiles</a><br/><br/>'
)
if is_download_on():
self.response.out.write(
'<a href="/gt/admin/turn_download_off">Turn OSM Download OFF</a><br/><br/>'
)
else:
self.response.out.write(
'<a href="/gt/admin/turn_download_on">Turn OSM Download ON</a><br/><br/>'
)
if action:
if action == 'create_geo_trees':
gt = GeoTree.get(gt_key_name='osm')
if not gt:
gt = GeoTree(key_name='osm',
max_z=config.max_z_osm,
min_z=config.min_z_osm)
gt.put()
self.response.out.write('\n\nInfo: Created osm GeoTree.')
else:
gt.max_z = config.max_z_osm
gt.min_z = config.min_z_osm
gt.put()
self.response.out.write('\n\nInfo: OSM GeoTree exists.')
gt = GeoTree.get(gt_key_name='cities')
if not gt:
gt = GeoTree(key_name='cities',
max_z=config.max_z_cities,
min_z=config.min_z_cities)
gt.put()
self.response.out.write('\nInfo: Created cities GeoTree.')
else:
gt.max_z = config.max_z_cities
gt.min_z = config.min_z_cities
gt.put()
self.response.out.write('\nInfo: Cities GeoTree exists.')
if action == 'add_points':
batch = OSMPOI.all().filter('is_in_tree =',
False).fetch(self._BATCH_ADD_SIZE)
if batch:
GeoTree.insert_points_list(batch,
max_z=17,
gt_key_name="osm")
self.response.out.write('\n\nInfo: added %d points' %
len(batch))
for p in batch:
p.is_in_tree = True
db.put(batch)
taskqueue.add(url='/gt/admin/add_points', method='GET')
else:
if GeoTree.exists(gt_key_name="osm"):
self.response.out.write('\n\nInfo: no POIs to add.')
taskqueue.add(url='/gt/admin/update_tiles',
method='GET')
else:
self.response.out.write(
'\n\nInfo: GeoTree does not exist.')
if action == 'add_cities':
batch = City.all().filter('is_in_tree =',
False).fetch(self._BATCH_ADD_SIZE)
if batch:
GeoTree.insert_points_list(batch, gt_key_name="cities")
self.response.out.write('\n\nInfo: added %d cities' %
len(batch))
for p in batch:
p.is_in_tree = True
db.put(batch)
else:
if GeoTree.exists(gt_key_name="cities"):
self.response.out.write(
'\n\nInfo: no cities left out of tree')
else:
self.response.out.write(
'\n\nInfo: GeoTree does not exist')
if action == 'update_tiles':
message = GeoTree.update_tiles(count=self._BATCH_UPDATE_SIZE,
gt_key_name="osm")
if message:
if 'nothing to update' in message:
self.response.out.write('<br/>' + message)
else:
taskqueue.add(url='/gt/admin/update_tiles', method='GET')
if action == 'update_cities_tiles':
message = GeoTree.update_tiles(count=self._BATCH_UPDATE_SIZE,
gt_key_name="cities")
if message:
self.response.out.write('<br/>' + message)
else:
self.response.out.write('\n\nInfo: updated tiles')
# memcaching is not used at the moment
if action == 'clear_cache':
memcache.flush_all()
self.response.out.write(
'<br/>All memcache entries are deleted.')
def get(self):
# Creating an entity with a system ID
e1 = Entity()
e1.prop = 1
e1.put()
k1 = e1.key()
self.response.write('<p>Entity e1 has a system ID = %d</p>' % k1.id())
# Creating an entity with a key name
e2 = Entity(key_name='alphabeta')
e2.prop = 2
e2.put()
k2 = e2.key()
self.response.write('<p>Entity e2 has a key name = %s</p>' % k2.name())
# Getting an entity by a known key
k = db.Key.from_path('Entity', 'alphabeta')
result = db.get(k)
# You could also use this shortcut:
# result = Entity.get_by_key_name('alphabeta')
if result:
self.response.write('<p>Got an entity by key name, prop = %d</p>' %
result.prop)
else:
self.response.write('<p>Could not find an entity with key name '
'"alphabeta"</p>')
# Inspecting entity objects
e3 = Entity()
assert (not e3.is_saved())
e3.put()
assert (e3.is_saved())
assert (not hasattr(e3, 'prop'))
e3.prop = 3
assert (hasattr(e3, 'prop'))
for n in range(1, 10):
value = n * n
setattr(e3, 'prop' + str(n), value)
assert (getattr(e3, 'prop' + str(7)) == 49)
self.response.write('<p>Properties of e3:</p><ul>')
for name in e3.instance_properties():
value = getattr(e3, name)
self.response.write('<li>%s = %d</li>' % (name, value))
self.response.write('</ul>')
# Batch put
e4 = Entity()
e5 = Entity()
db.put([e4, e5])
# Delete an entity using the model object
db.delete(e1)
e2.delete()
# Delete by key
k = db.Key.from_path('Entity', 'alphabeta')
db.delete(k)
# Batch delete
db.delete([e4, e5])
self.response.write('<p>Entities deleted.</p>')
self.response.write('<p>The time is: %s</p>' %
str(datetime.datetime.now()))
def test_search_by_name_only(self):
db.put(self.p1)
db.put(self.p2)
db.put(self.p3)
db.put(self.p4)
db.put(self.p5)
db.put(self.p6)
db.put(self.p7)
db.put(self.p8)
db.put(self.p9)
db.put(self.p10)
db.put(self.p11)
db.put(self.p12)
db.put(self.p13)
db.put(self.p14)
db.put(self.p15)
db.put(self.p16)
db.put(self.p17)
db.put(self.p18)
full_text_search.add_record_to_index(self.p1)
full_text_search.add_record_to_index(self.p2)
full_text_search.add_record_to_index(self.p3)
full_text_search.add_record_to_index(self.p4)
full_text_search.add_record_to_index(self.p5)
full_text_search.add_record_to_index(self.p6)
full_text_search.add_record_to_index(self.p7)
full_text_search.add_record_to_index(self.p8)
full_text_search.add_record_to_index(self.p9)
full_text_search.add_record_to_index(self.p10)
full_text_search.add_record_to_index(self.p11)
full_text_search.add_record_to_index(self.p12)
full_text_search.add_record_to_index(self.p13)
full_text_search.add_record_to_index(self.p14)
full_text_search.add_record_to_index(self.p15)
full_text_search.add_record_to_index(self.p16)
full_text_search.add_record_to_index(self.p17)
full_text_search.add_record_to_index(self.p18)
# Search by alternate name
results = full_text_search.search('haiti', 'Iorin', 5)
assert set([r.record_id for r in results]) == \
set(['haiti/0505'])
# Search by family name
results = full_text_search.search('haiti', 'Minase', 5)
assert set([r.record_id for r in results]) == \
set(['haiti/0505'])
# Search by given name
results = full_text_search.search('haiti', 'Iori', 5)
assert set([r.record_id for r in results]) == \
set(['haiti/0505'])
# Search by given name + family name
results = full_text_search.search('haiti', 'Minase Iori', 5)
assert set([r.record_id for r in results]) == \
set(['haiti/0505'])
# Search by full name
resutls = full_text_search.search('haiti', 'Iori Minase', 5)
assert set([r.record_id for r in results]) == \
set(['haiti/0505'])
# Search by name & location
results = full_text_search.search('haiti', 'Chihaya Arao', 5)
assert set([r.record_id for r in results]) == \
set(['haiti/0225'])
# Search Cyrillic record by name & location
results = full_text_search.search('haiti', 'Ritsuko Tottori', 5)
assert set([r.record_id for r in results]) == \
set(['haiti/0623'])
# Search by home_street only
results = full_text_search.search('haiti', 'Kunaideme72', 5)
assert not results
# Search by home_city only
results = full_text_search.search('haiti', 'Arao', 5)
assert not results
# Search by home_state only
results = full_text_search.search('haiti', 'Kumamoto', 5)
assert not results
# Search by home_postal_code only
results = full_text_search.search('haiti', '864-0003', 5)
assert not results
# Search by home_neighborhood only
results = full_text_search.search('haiti', 'Araokeibajou', 5)
assert not results
# Search by home_country only
results = full_text_search.search('haiti', 'Japan', 5)
assert not results
# Search in a different repository
results = full_text_search.search('japan', 'Iori', 5)
assert not results
# Check no results
results = full_text_search.search('haiti', 'Producer san', 5)
assert not results
# Search with no query text
results = full_text_search.search('haiti', '', 5)
assert not results
# Search deleted record
delete.delete_person(self, self.p5)
results = full_text_search.search('haiti', 'Ami', 5)
assert not results
# Check rank order (name match heigher than location match)
results = full_text_search.search('haiti', 'Rin Shibuya', 5)
assert [r.record_id for r in results] == \
['haiti:0810', 'haiti:0203']
# Search romaji record by kanji name
results = full_text_search.search('haiti', u'千早', 5)
assert set([r.record_id for r in results]) == \
set(['haiti/0225'])
# Search romaji record by kanji name and location
results = full_text_search.search('haiti', u'千早 荒尾', 5)
assert set([r.record_id for r in results]) == \
set(['haiti/0225'])
# Check rank order
# (same kanji higher than different kanji with the same reading)
results = full_text_search.search('haiti', u'菊地 真', 5)
assert [r.record_id for r in results] == \
['haiti/0829', 'haiti/1829']
results = full_text_search.search('haiti', u'菊地 眞', 5)
assert [r.record_id for r in results] == \
['haiti/1829', 'haiti/0829']
# Search kanji record by multi reading
results = full_text_search.search('haiti', u'hagiwara', 5)
assert set([r.record_id for r in results]) == \
set(['haiti/1224'])
results = full_text_search.search('haiti', u'ogiwara', 5)
assert set([r.record_id for r in results]) == \
set(['haiti/1224'])
# Search romaji record by hiragana name and location
results = full_text_search.search('haiti', u'ちはや あらお', 5)
assert set([r.record_id for r in results]) == \
set(['haiti/0225'])
# Search by full name without space
results = full_text_search.search('haiti', 'HibikiGanaha', 5)
assert set([r.record_id for r in results]) == \
set(['haiti/1010'])
# Search kanji record by full name without space
results = full_text_search.search('haiti', u'AzusaMiura', 5)
assert set([r.record_id for r in results]) == \
set(['haiti/0719'])
# Search Cyrillic record by full name without space
results = full_text_search.search('haiti', u'RitsukoAkiduki', 5)
assert set([r.record_id for r in results]) == \
set(['haiti/0623'])
# Search full name without space record by given name and family name
results = full_text_search.search('haiti', u'Kotori Otonashi', 5)
assert set([r.record_id for r in results]) == \
set(['haiti/0909'])
# Search Cyrillic record by full name without space
results = full_text_search.search('haiti', u'OtonashiKotori', 5)
assert set([r.record_id for r in results]) == \
set(['haiti/0909'])
# Search Chinese record by kanji
results = full_text_search.search('haiti', u'真美', 5)
assert set([r.record_id for r in results]) == \
set(['haiti/0523'])
# Search by Special Chinese Family Name
# while records are written in English
results = full_text_search.search('haiti', u'单鱼', 5)
assert set([r.record_id for r in results]) == \
set(['haiti/0911'])
# Search by Pinyin(Chinese Romaji)
# while records are written in Chinese
results = full_text_search.search('haiti', u'Zeng Cheng', 5)
assert set([r.record_id for r in results]) == \
set(['haiti/0910'])
# Search by Chinese
# while records are written in Chinese
results = full_text_search.search('haiti', u'曾诚', 5)
assert set([r.record_id for r in results]) == \
set(['haiti/0910'])
def test_clean_up_in_test_mode(self):
"""Test the clean up in test mode."""
def run_clean_up_in_test_mode_task():
"""Runs the CleanUpInTestMode task."""
test_handler.initialize_handler(
tasks.CleanUpInTestMode, tasks.CleanUpInTestMode.ACTION).get()
tasks.CleanUpInTestMode.DELETION_AGE_SECONDS = 2 * 3600 # 2 hours
# entry_date of p3 is 4 hours after p1 and p2.
self.p3 = model.Person.create_original(
'haiti',
first_name='Taro',
last_name='Google',
home_street='Roppongi',
home_city='Minato',
home_state='Tokyo',
source_date=datetime.datetime(2010, 1, 1),
entry_date=datetime.datetime(2010, 1, 1, 4, 0, 0),
expiry_date=datetime.datetime(2010, 3, 1),
other='')
self.key_p3 = db.put(self.p3)
self.to_delete.append(self.p3)
# Initial state: three Persons and one Note.
assert model.Person.all().count() == 3
assert model.Note.get('haiti', self.note_id)
assert db.get(self.photo_key)
assert db.get(self.key_p1).is_expired == False # still exists
assert db.get(self.key_p2).is_expired == False
assert db.get(self.key_p3).is_expired == False
# verify schedule_next_task does the right thing.
utcnow = datetime.datetime(2010, 1, 1)
config.set(test_mode=True, repo='haiti')
query = model.Person.all()
query.get()
self.mox = mox.Mox()
self.mox.StubOutWithMock(taskqueue, 'add')
taskqueue.add(method='GET',
url='/haiti/tasks/clean_up_in_test_mode',
params={
'cursor': query.cursor(),
'utcnow':
str(calendar.timegm(utcnow.utctimetuple())),
'queue_name': 'clean_up_in_test_mode',
},
name=mox.IsA(str))
self.mox.ReplayAll()
cleanup = \
test_handler.initialize_handler(tasks.CleanUpInTestMode,
tasks.CleanUpInTestMode.ACTION)
cleanup.schedule_next_task(query.cursor(), utcnow)
self.mox.UnsetStubs()
self.mox.VerifyAll()
# Nothing happens if test_mode is False.
config.set(test_mode=False, repo='haiti')
set_utcnow_for_test(datetime.datetime(2010, 6, 1))
run_clean_up_in_test_mode_task()
assert db.get(self.key_p1).is_expired == False # still exists
assert db.get(self.key_p2).is_expired == False
assert db.get(self.key_p3).is_expired == False
# Records with entry_date before 2010-01-01 3:00 are deleted.
config.set(test_mode=True, repo='haiti')
set_utcnow_for_test(datetime.datetime(2010, 1, 1, 5, 0, 0))
run_clean_up_in_test_mode_task()
assert db.get(self.key_p1) is None
assert db.get(self.key_p2) is None
assert db.get(self.key_p3).is_expired == False # still exists
# All records are deleted.
config.set(test_mode=True, repo='haiti')
set_utcnow_for_test(datetime.datetime(2010, 1, 1, 7, 0, 0))
run_clean_up_in_test_mode_task()
assert db.get(self.key_p1) is None
assert db.get(self.key_p2) is None
assert db.get(self.key_p3) is None
def _save_notification_and_payload(cls, notification, payload):
return db.put([notification, payload])
def test_delete_record_from_index(self):
db.put(self.p4)
full_text_search.add_record_to_index(self.p4)
full_text_search.delete_record_from_index(self.p4)
results = full_text_search.search('haiti', 'Miki', 5)
assert not results
def trans():
sc.provisioned = False
sc.deleted = True
sln_settings.updates_pending = True
db.put([sln_settings, sc])
return sln_settings
def updateTasksPostStudentSignUp(request, *args, **kwargs):
"""Appengine task that updates the GHOP Tasks after the student signs up.
Expects the following to be present in the POST dict:
student_key: Specifies the student key name who registered
Args:
request: Django Request object
"""
from soc.modules.ghop.logic.models import student as ghop_student_logic
post_dict = request.POST
student_key = post_dict.get('student_key')
if not student_key:
# invalid student data, log and return OK
return error_handler.logErrorAndReturnOK('Invalid Student data: %s' %
post_dict)
student_entity = ghop_student_logic.logic.getFromKeyNameOr404(student_key)
# retrieve all tasks currently assigned to the user
task_fields = {
'user': student_entity.user,
}
task_entities = ghop_task_logic.logic.getForFields(task_fields)
# TODO(madhusudan) move this to the Task Logic
# Make sure the tasks store references to the student as well as
# closing all tasks that are AwaitingRegistration.
for task_entity in task_entities:
task_entity.student = student_entity
if task_entity.status == 'AwaitingRegistration':
task_entities.remove(task_entity)
properties = {'status': 'Closed'}
changes = [
ugettext('User-MelangeAutomatic'),
ugettext('Action-Student registered'),
ugettext('Status-%s' % (properties['status']))
]
comment_properties = {
'parent':
task_entity,
'scope_path':
task_entity.key().name(),
'created_by':
None,
'changes':
changes,
'content':
ugettext(
'(The Melange Automated System has detected that the student '
'has signed up for the program and hence has closed this task.'
),
}
ghop_task_logic.logic.updateEntityPropertiesWithCWS(
task_entity, properties, comment_properties)
db.put(task_entities)
# return OK
return http.HttpResponse()
e = Employee(name="",
role="manager",
account=users.get_current_user())
e.hire_date = datetime.datetime.now()
e.put()
# other example, consulta SQL
training_registration_list = [users.User("*****@*****.**"),
users.User("*****@*****.**"),
users.User("*****@*****.**")]
employees_trained = db.GqlQuery("SELECT * FROM Employee WHERE account IN :1",
training_registration_list)
for e in employees_trained:
e.new_hire_training_completed = True
db.put(e)
# poliformismo
from google.appengine.ext import db
from google.appengine.ext.db import polymodel
class Contact(polymodel.PolyModel):
phone_number = db.PhoneNumberProperty()
address = db.PostalAddressProperty()
class Person(Contact):
first_name = db.StringProperty()
last_name = db.StringProperty()
mobile_number = db.PhoneNumberProperty()
class Company(Contact):
COUNTER_SEND_MAIL_TASK_FAILED.inc()
# Set by except: clause above. pylint: disable=raising-bad-type
raise exception
if sent:
cls._mark_sent(notification, now)
if failed_permanently:
cls._mark_failed(notification, now, exception, permanent=True)
if sent or failed_permanently:
policy.run(notification, payload)
cls._mark_done(notification, now)
db.put([notification, payload])
COUNTER_RETENTION_POLICY_RUN.inc()
if sent:
COUNTER_SEND_MAIL_TASK_SENT.inc()
elif failed_permanently:
COUNTER_SEND_MAIL_TASK_FAILED_PERMANENTLY.inc()
COUNTER_SEND_MAIL_TASK_SUCCESS.inc()
@classmethod
@db.transactional(
propagation=datastore_rpc.TransactionOptions.INDEPENDENT, xg=True)
def _record_failure(
cls, notification, payload, exception, dt=None, permanent=False,
def post(self):
track = self.request.get('track')
bestlaps = {}
for bl in BestLap.all().filter('track =', track):
if bl.isBest is True:
bestlaps[bl.raceclass.name] = bl
upload_files = self.get_uploads('file')[0]
blob_key = upload_files.key()
blob_info = upload_files
record = Record(csv=str(blob_info)).put()
blob_reader = blobstore.BlobReader(blob_key)
reader = csv.DictReader(blob_reader)
lapsToUpload = []
for row in reader:
#row = row.replace('"','').replace(', ', ' ').strip()
if 'Best Tm' in row:
time = row['Best Tm']
else:
time = row['Overall BestTm']
if 'Laps' in row:
laps = row['Laps']
else:
laps = row['Appeared']
position = row['Pos']
point_in_class = row['PIC']
carnum = row['No.']
racer_name = normalize_string(row['Name'])
racer_class = normalize_string(row['Class'])
diff = row['Diff']
gap = row['Gap']
points = row['Points']
car_make = normalize_string(row['Make'])
car_model = normalize_string(row['Model'])
car_year = row['Year']
car_color = row['Color']
city = row['City']
state = row['State']
sponsor = normalize_string(row['Sponsor'])
email = row['Email']
if time.count(':') == 0 and time:
time = '0:' + time
if validLap(racer_class, point_in_class):
pt = process_time(time)
t = track #Track.get_or_insert(key_name=self.request.get('track'), name=self.request.get('track'), lap_distance=1.02)
g = self.request.get('group')
sd = self.request.get('date')
dt = datetime.strptime(sd, '%Y-%m-%d')
tr = Track.get_or_insert(key_name=t, name=t)
e = Event.get_or_insert(key_name=g+t+sd, name=g+t, track=tr, date=dt)
c = Car.get_or_insert(key_name=carnum+car_make+car_model+car_color+car_year, make=car_make, model=car_model,year=car_year,color=car_color,number=carnum)
cl = RaceClass.get_or_insert(key_name=racer_class, name=racer_class)
if email:
email
else:
email = racer_name.split()[0].lower() + racer_name.split()[1].lower()+'@gmail.com'
r = Racer.get_or_insert(key_name=racer_name.split()[0][0:1].lower() + racer_name.split()[1].lower(), email=email,name=racer_name, driver=users.User(email), points=int(points), car=c, raceclass=cl)
if sponsor:
r.sponsor=Sponsor.get_or_insert(key_name=sponsor, name=sponsor)
r.put()
best = BestLap.get_or_insert(key_name=sd+t+g+cl.name+racer_name.replace(' ','.'), driver=r, raceclass=cl, track=t, time=pt, event=e, isBest=False, date=dt)
if cl.name in bestlaps:
if pt < bestlaps[cl.name].time and pt != 0.0:
print str(pt) + ' is better than ' + bestlaps[cl.name].driver.name + 's time of ' + str(bestlaps[cl.name].time)
best.isBest = True #Mark current record as best
bestlaps[cl.name].isBest = False #Mark old record as not best
bestlaps[cl.name].put() #Commit old record to db
bestlaps[cl.name] = best #Replace record in local dictionary with new best record for class
elif pt != 0.0:
best.isBest = True
bestlaps[cl.name] = best
lapsToUpload.append(best)
db.put(lapsToUpload)
self.redirect('/')
def post(self):
try:
#global global_dict
#global_dict = {}
starttime = time.time()
cutoff_date = datetime.datetime.now() + datetime.timedelta(-365)
cutoff_date_string = cutoff_date.strftime("%Y-%m-%d %H:%M:%S")
parts = self.request.body.split("&")
requests = {}
if parts is not None and parts[0] != "":
for pair in parts:
ab = pair.split('=')
requests[ab[0]] = ab[1]
force = bool(requests.get("force", False))
write = bool(requests.get("write", False))
minwrite = bool(requests.get("minwrite", False))
rpcList = []
client = memcache.Client()
q = structures.Rumble.all()
rumbles = []
for r in q.run():
memr = memcache.get(r.Name)
if memr is not None:
r = memr
if r.BatchScoresAccurate and not force:
continue
rumbles.append(r)
for r in rumbles:
scoresdicts = pickle.loads(
zlib.decompress(r.ParticipantsScores))
entries = len(scoresdicts)
r.__dict__["entries"] = entries
rumbles.sort(key=lambda r: -r.__dict__["entries"])
first = True
for r in rumbles:
if not first:
time.sleep(5)
gc.collect()
gc.collect(2)
first = False
logging.info("mem usage at start of " + r.Name + ": " +
str(runtime.memory_usage().current()) + "MB")
try:
scores = pickle.loads(zlib.decompress(
r.ParticipantsScores))
except:
scoresdicts = marshal.loads(
zlib.decompress(r.ParticipantsScores))
scoreslist = [structures.LiteBot() for _ in scoresdicts]
for s, d in zip(scoreslist, scoresdicts):
s.__dict__.update(d)
scores = {s.Name: s for s in scoreslist}
if len(scores) == 0:
continue
r.ParticipantsScores = None
#gc.collect()
particHash = [p + "|" + r.Name for p in scores]
particSplit = list_split(particHash, 32)
ppDict = {}
for l in particSplit:
ppDict.update(memcache.get_multi(l))
time.sleep(0.1)
particSplit = None
bots = [ppDict.get(h, None) for h in particHash]
botsdict = {}
missingHashes = []
missingIndexes = []
for i in xrange(len(bots)):
if bots[i] is None:
missingHashes.append(particHash[i])
missingIndexes.append(i)
elif isinstance(bots[i], structures.BotEntry):
bots[i] = structures.CachedBotEntry(bots[i])
if len(missingHashes) > 0:
bmis = structures.BotEntry.get_by_key_name(missingHashes)
#lost = False
lostList = []
for i in xrange(len(missingHashes)):
if bmis[i] is not None:
cb = structures.CachedBotEntry(bmis[i])
bots[missingIndexes[i]] = cb
botsdict[missingHashes[i]] = cb
else:
bots[missingIndexes[i]] = None
lostList.append(missingHashes[i])
#lost = True
while len(particHash) > 0:
particHash.pop()
particHash = None
while len(missingHashes) > 0:
missingHashes.pop()
missingHashes = None
while len(missingIndexes) > 0:
missingIndexes.pop()
missingIndexes = None
logging.info("mem usage after loading bots: " +
str(runtime.memory_usage().current()) + "MB")
bots = filter(lambda b: b is not None, bots)
get_key = attrgetter("APS")
bots.sort(key=lambda b: get_key(b), reverse=True)
gc.collect()
botIndexes = {}
for i, b in enumerate(bots):
b.Name = b.Name.encode('ascii')
intern(b.Name)
botIndexes[b.Name] = i
b.VoteScore = 0.
botlen = len(bots)
APSs = numpy.empty([botlen, botlen])
APSs.fill(numpy.nan)
totalAlivePairs = 0
for i, b in enumerate(bots):
try:
pairings = pickle.loads(zlib.decompress(
b.PairingsList))
except:
pairsDicts = marshal.loads(
zlib.decompress(b.PairingsList))
pairings = [structures.ScoreSet() for _ in pairsDicts]
for s, d in zip(pairings, pairsDicts):
s.__dict__.update(d)
removes = []
alivePairings = 0
for q, p in enumerate(pairings):
j = botIndexes.get(p.Name, -1)
if j != -1:
APSs[j, i] = numpy.float64(p.APS)
p.Alive = True
alivePairings += 1
else:
removes.append(q)
b.Pairings = alivePairings
totalAlivePairs += alivePairings
removes.reverse()
removed = False
for q in removes:
p = pairings[q]
if p.LastUpload < cutoff_date_string:
removed = True
pairings.pop(q)
else:
if p.Alive:
removed = True
p.Alive = False
if removed:
b.PairingsList = zlib.compress(
pickle.dumps(pairings, -1), 1)
gc.collect()
APSs += numpy.float64(100) - APSs.transpose()
APSs *= numpy.float64(0.5)
numpy.fill_diagonal(APSs, numpy.nan)
gc.collect()
logging.info(
str(len(bots)) + " bots loaded, total of " +
str(totalAlivePairs) + " alive pairings")
logging.info("mem usage after unzipping pairings: " +
str(runtime.memory_usage().current()) + "MB")
#Vote
mins = numpy.nanmax(APSs, 1)
for i, minimum in enumerate(mins):
minIndexes = numpy.argwhere(APSs[i, ...] == minimum)
ties = len(minIndexes)
if ties > 0:
increment = 1. / ties
for minIndex in minIndexes:
bots[minIndex].VoteScore += increment
#inv_len = 1.0/botlen
for b in bots:
if b.Pairings > 0:
b.VoteScore = 100.0 * b.VoteScore / float(b.Pairings)
else:
b.VoteScore = 0
#KNN PBI
half_k = int(math.ceil(math.sqrt(botlen) / 2))
KNN_PBI = -numpy.ones((botlen, botlen))
for i in xrange(len(bots)):
low_bound = max([0, i - half_k])
high_bound = min([botlen - 1, i + half_k])
low_high_bound = min([i + 1, high_bound])
before = APSs[:, low_bound:i]
after = APSs[:, low_high_bound:high_bound]
compare = numpy.hstack((before, after))
mm = numpy.mean(numpy.ma.masked_array(
compare, numpy.isnan(compare)),
axis=1)
KNN_PBI[:, i] = APSs[:, i] - mm.filled(numpy.nan)
# a[i] = 0
# logging.info("mean error of transpose: " + str(numpy.mean(numpy.square(a))))
#KNN_PBI[KNN_PBI == numpy.nan] = -1
#logging.info("mem usage after KNNPBI: " + str(runtime.memory_usage().current()) + "MB")
# Avg Normalised Pairing Percentage
mins = numpy.nanmin(APSs, 1)
maxs = numpy.nanmax(APSs, 1)
inv_ranges = numpy.float64(1.0) / (maxs - mins)
NPPs = -numpy.ones((botlen, botlen))
for i in range(botlen):
if numpy.isfinite(inv_ranges[i]):
NPPs[i, :] = numpy.float64(100) * (
APSs[i, :] - mins[i]) * inv_ranges[i]
else:
NPPs[i, :] = numpy.float64(100)
#NPPs[NPPs] = -1
#logging.info("mem usage after ANPP: " + str(runtime.memory_usage().current()) + "MB")
changedBots = [] #bots with new pairings since last run
# save to cache
botsdict = {}
for i, b in enumerate(bots):
# try:
pairings = pickle.loads(zlib.decompress(b.PairingsList))
# except:
# pairsDicts = marshal.loads(zlib.decompress(b.PairingsList))
#
# pairings = [structures.ScoreSet() for _ in pairsDicts]
# for s,d in zip(pairings,pairsDicts):
# s.__dict__.update(d)
nppCount = 0
totalNPP = 0.0
apsCount = 0
totalAPS = 0.0
aliveCount = 0
changed = False
for p in pairings:
j = botIndexes.get(p.Name, -1)
if j != -1:
p.Alive = True
changePotential = (p.KNNPBI == 0.0 and p.NPP == -1)
aliveCount += 1
p.KNNPBI = float(KNN_PBI[j, i])
p.NPP = float(NPPs[j, i])
if not numpy.isnan(APSs[j, i]):
p.APS = float(APSs[j, i])
totalAPS += p.APS
apsCount += 1
if numpy.isnan(p.KNNPBI):
p.KNNPBI = 0
if numpy.isnan(p.NPP):
p.NPP = -1
else:
totalNPP += p.NPP
nppCount += 1
if changePotential and p.KNNPBI != 0.0 and p.NPP != -1:
changed = True
else:
p.Alive = False
p.KNNPBI = 0
p.NPP = -1
if nppCount > 0:
b.ANPP = float(totalNPP / nppCount)
else:
b.ANPP = -1.0
if apsCount > 0:
b.APS = float(totalAPS / apsCount)
else:
b.APS = -1.0
b.PairingsList = zlib.compress(pickle.dumps(pairings, -1),
1)
b.Pairings = aliveCount
if b.Pairings > 0:
botsdict[b.key_name] = b
if changed:
changedBots.append(b)
KNN_PBI = None
APSs = None
NPPs = None
logging.info("mem usage after zipping: " +
str(runtime.memory_usage().current()) + "MB")
gc.collect()
#logging.info("mem usage after gc: " + str(runtime.memory_usage().current()) + "MB")
if len(botsdict) > 0:
splitlist = dict_split(botsdict, 20)
logging.info("split bots into " + str(len(splitlist)) +
" sections")
for d in splitlist:
rpcList.append(client.set_multi_async(d))
time.sleep(.5) #throttle
logging.info("wrote " + str(len(botsdict)) +
" bots to memcache")
botsdict.clear()
botsdict = None
scores = {b.Name: structures.LiteBot(b) for b in bots}
# bots = None
r.ParticipantsScores = None
gc.collect()
r.ParticipantsScores = db.Blob(
zlib.compress(
pickle.dumps(scores, pickle.HIGHEST_PROTOCOL), 3))
logging.info("mem usage after participants zipping: " +
str(runtime.memory_usage().current()) + "MB")
#r.ParticipantsScores = zlib.compress(marshal.dumps([scores[s].__dict__ for s in scores]),4)
scores = None
if write:
writebots = [None] * len(bots)
for i, b in enumerate(bots):
putb = structures.BotEntry(key_name=b.key_name)
putb.init_from_cache(b)
writebots[i] = putb
write_lists = list_split(writebots, 50)
for subset in write_lists:
db.put(subset)
time.sleep(0.1) #throttle
logging.info("wrote " + str(len(writebots)) +
" bots to database")
while len(bots) > 0:
bots.pop()
bots = None
if minwrite:
writebots = [None] * len(changedBots)
for i, b in enumerate(changedBots):
putb = structures.BotEntry(key_name=b.key_name)
putb.init_from_cache(b)
writebots[i] = putb
write_lists = list_split(writebots, 50)
for subset in write_lists:
db.put(subset)
time.sleep(0.1)
logging.info("wrote " + str(len(writebots)) +
" changed bots to database")
while len(changedBots) > 0:
changedBots.pop()
changedBots = None
gc.collect()
if write or minwrite:
r.BatchScoresAccurate = True
rpcList.append(client.set_multi_async({r.Name: r}))
db.put([r])
#gc.collect()
r = None
logging.info("mem usage after write: " +
str(runtime.memory_usage().current()) + "MB")
for rpc in rpcList:
rpc.get_result()
elapsed = time.time() - starttime
logging.info("Success in " + str(round(1000 * elapsed) / 1000) +
"s")
self.response.out.write("Success in " +
str(round(1000 * elapsed)) + "ms")
except:
logging.exception('')
elapsed = time.time() - starttime
logging.info("Error in " + str(round(1000 * elapsed) / 1000) + "s")
self.response.out.write("Error in " + str(round(1000 * elapsed)) +
"ms")
def CreateMachines(num_instances,
token,
os,
browser,
browser_version,
download_info,
retries=0):
"""Create and launch EC2 machines for the given parameters.
Args:
num_instances: An integer representing the number of instances to spawn
with the given configuration.
token: A string representing the token for this run.
os: An integer that corresponds to an enum.OS value.
browser: An integer that corresponds to an enum.BROWSER value.
browser_version: A string representing browser version to use.
Specifically, this should be the channel for Chrome.
download_info: A string representing the information necessary for
calculating the version and browser download url for the given machine.
retries: An optional paramater specifying the initial retry count for the
machine.
"""
logging.info(
'\n'.join([
'num_instances: %d', 'token: %s', 'os: %d', 'browser: %d',
'browser_version: %s'
]), num_instances, token, os, browser, browser_version)
ec2 = ec2_manager.EC2Manager()
user_data = simplejson.dumps({
'channel': browser_version,
'os': OS_TO_USER_DATA[os],
'token': token,
'download_info': download_info
})
logging.info('Spawning EC2 machines.')
# Note: All exceptions are caught here because the EC2 API could fail after
# successfully starting a machine. Because this task is rescheduled on
# failure, we need to make sure we don't keep spawning EC2 machines.
try:
bots_instances = ec2.StartAmiWithOs(
os,
count=num_instances,
instance_type=DEFAULT_INSTANCE_SIZE,
user_data=user_data)
except Exception:
logging.exception('Something failed when setting up the EC2 instance. '
'Stopping setup for this instance.')
return
logging.info('Creating the corresponding ClientMachine models.')
new_instances = []
for instance in bots_instances:
new_instances.append(
client_machine.ClientMachine(
vm_service=enum.VM_SERVICE.EC2,
os=os,
browser=browser,
browser_version=browser_version,
client_id=instance.inst_id,
status=enum.MACHINE_STATUS.PROVISIONED,
retry_count=retries,
token=token,
download_info=download_info))
db.put(new_instances)
logging.info('Finished creating the ClientMachine models.')
def _mock_data(self):
# The fictional day for these tests Wednesday, July 29, 2015
slacklib._TODAY_FN = lambda: datetime.datetime(2015, 7, 29)
# Stuart created his account, but has never once filled out a snippet
db.put(models.User(email='*****@*****.**'))
# Fleetwood has two recent snippets, and always uses markdown lists,
# but sometimes uses different list indicators or indention.
db.put(models.User(email='*****@*****.**'))
db.put(
models.Snippet(email='*****@*****.**',
week=datetime.date(2015, 7, 27),
text=textwrap.dedent("""
* went for a walk
* sniffed some things
* hoping to sniff more things! #yolo
""")))
db.put(
models.Snippet(email='*****@*****.**',
week=datetime.date(2015, 7, 20),
text=textwrap.dedent("""
- lots of walks this week
- not enough sniffing, hope to remedy next week!
""")))
# Toby has filled out two snippets, but missed a week in-between while
# on vacation. When he got back from vacation he was still jetlagged so
# he wrote a longform paragraph instead of a list.
db.put(models.User(email='*****@*****.**'))
db.put(
models.Snippet(email='*****@*****.**',
week=datetime.date(2015, 7, 13),
text=textwrap.dedent("""
- going on vacation next week, so excited!
""")))
db.put(
models.Snippet(email='*****@*****.**',
week=datetime.date(2015, 7, 27),
text=textwrap.dedent("""
I JUST GOT BACK FROM VACATION IT WAS TOTALLY AWESOME AND I SNIFFED
ALL SORTS OF THINGS. I GUESS I NEED TO WRITE SOMETHING HERE, HUH?
OK THEN:
- I had fun.
LUNCHTIME SUCKERS!
""")))
# Fozzie tried hard to create an entry manually in the previous week,
# but didn't understand markdown list syntax and got discouraged (so
# has no entry this week, and a malformed one last week).
db.put(models.User(email='*****@*****.**'))
db.put(
models.Snippet(email='*****@*****.**',
week=datetime.date(2015, 7, 20),
text=textwrap.dedent("""
-is this how I list?
-why is it not formatting??!?
""")))
def get(self):
query = VideoPlaylist.all()
all_video_playlists = query.fetch(100000)
for video_playlist in all_video_playlists:
video_playlist.live_association = False
db.put(all_video_playlists)
def put_google_events(service_user, calendar_str_key, calendar_id, solution,
google_calendar_finished, google_events, language):
to_put = []
no_title_text = common_translate(language, SOLUTION_COMMON, '(No title)')
for google_event in google_events:
try:
google_event_id = google_event["id"]
event_parent_key = parent_key(service_user, solution)
event = Event.all().ancestor(event_parent_key).filter(
"source =",
Event.SOURCE_GOOGLE_CALENDAR).filter("external_id =",
google_event_id).get()
if not event:
event = Event(parent=event_parent_key,
source=Event.SOURCE_GOOGLE_CALENDAR,
external_id=google_event_id)
if google_event.get("status", "confirmed") == "cancelled":
if not event.is_saved():
continue
event.deleted = True
elif not event.deleted:
event.deleted = False
event.title = google_event.get('summary', no_title_text)
event.place = google_event.get("location",
u"").replace(u"\n", u" ")
event.organizer = google_event.get("organizer",
{}).get("displayName", u"")
event.description = google_event.get('description', u"")
event.calendar_id = calendar_id
event.external_link = google_event["htmlLink"]
event.start_dates = list()
event.end_dates = list()
if google_event["start"].get("dateTime", None):
start_date_with_tz = dateutil.parser.parse(
google_event["start"]["dateTime"])
end_date_with_tz = dateutil.parser.parse(
google_event["end"]["dateTime"])
day_difference = abs(
(end_date_with_tz - start_date_with_tz).days)
if day_difference == 0:
start_epoch = get_epoch_from_datetime(
datetime(start_date_with_tz.year,
start_date_with_tz.month,
start_date_with_tz.day,
start_date_with_tz.hour,
start_date_with_tz.minute))
event.start_dates.append(start_epoch)
end_epoch = get_epoch_from_datetime(
datetime(end_date_with_tz.year, end_date_with_tz.month,
end_date_with_tz.day, end_date_with_tz.hour,
end_date_with_tz.minute))
end_date = datetime.fromtimestamp(end_epoch)
event.end_dates.append(
int(
timedelta(
hours=end_date.hour,
minutes=end_date.minute,
seconds=end_date.second).total_seconds()))
# TODO: multi day event
# else:
# start_date = datetime.strptime(google_event["start"]["date"], '%Y-%m-%d')
# end_date = datetime.strptime(google_event["end"]["date"], '%Y-%m-%d')
# day_difference = abs((end_date - start_date).days)
if event.start_dates:
event.first_start_date = event.start_dates[0]
event.last_start_date = event.start_dates[-1]
to_put.append(event)
else:
logging.info("Skipping event because it had no start_dates")
logging.debug(google_event)
except:
logging.warn('Failed to put Google Event: %s', google_event)
raise
if to_put:
db.put(to_put)
if google_calendar_finished:
def trans():
scgs = SolutionCalendarGoogleSync.get_by_key_name(
service_user.email())
if not scgs:
return True # update was run from saving a calendar
if calendar_str_key in scgs.google_calendar_keys:
scgs.google_calendar_keys.remove(calendar_str_key)
if not scgs.google_calendar_keys:
scgs.delete()
return True
scgs.put()
return False
if db.run_in_transaction(trans):
from solutions.common.bizz.provisioning import populate_identity_and_publish
sln_settings = get_solution_settings(service_user)
sln_main_branding = get_solution_main_branding(service_user)
deferred.defer(populate_identity_and_publish, sln_settings,
sln_main_branding.branding_key)
def taskDifficultyEditPost(self, request, program_entity, params):
"""View method for edit task difficulty tags POST requests.
"""
post_dict = request.POST
operation = simplejson.loads(post_dict.get('operation'))
# invalid request
INVALID_REQUEST_RESPONSE = http.HttpResponse()
INVALID_REQUEST_RESPONSE.status_code = 400
if not operation:
return INVALID_REQUEST_RESPONSE
op = operation.get('op')
# TODO(ljvderijk): How do we want to deal with the setting of the value
# property in the tag since it now requires an extra put.
data = operation['data']
if op == 'add':
for tag_data in data:
tag = gci_task_model.TaskDifficultyTag.get_or_create(
program_entity, tag_data['name'])
tag.value = int(tag_data['value'])
tag.put()
elif op == 'change':
current_tag_data = data[0]
new_tag_data = data[1]
current_tag_name = current_tag_data['name']
new_tag_name = new_tag_data['name']
current_tag = gci_task_model.TaskDifficultyTag.get_by_scope_and_name(
program_entity, current_tag_name)
if not current_tag:
return INVALID_REQUEST_RESPONSE
if current_tag_name != new_tag_name:
# rename tag
new_tag = gci_task_model.TaskDifficultyTag.copy_tag(
program_entity, current_tag_name, new_tag_name)
# TODO(ljvderijk): The tag copy method should work with new fields
new_tag.order = current_tag.order
new_tag.value = int(new_tag_data['value'])
new_tag.put()
else:
# change value of the tag
current_tag.value = int(new_tag_data['value'])
current_tag.put()
elif op == 'delete':
for tag_data in data:
gci_task_model.TaskDifficultyTag.delete_tag(
program_entity, tag_data['name'])
elif op == 'reorder':
tags = []
for i in range(0, len(data)):
tag_data = data[i]
tag = gci_task_model.TaskDifficultyTag.get_by_scope_and_name(
program_entity, tag_data['name'])
tag.order = i
tags.append(tag)
db.put(tags)
return http.HttpResponse()
def update_count(type, add=[], subtract=[]):
#input: type = "archive" or "tag"; add, subtract = e.g. ["200412","200501"] or ["Books", "Games"]
if type == "archive":
cls = Archive
elif type == "tag":
cls = Tag
items_dict = {}
for t in add:
if t in items_dict:
items_dict[t] += 1
else:
items_dict[t] = 1
for t in subtract:
if t in items_dict:
items_dict[t] -= 1
else:
items_dict[t] = -1
for k in items_dict.keys(
): # No need to modify if the count change to a term is 0.
if items_dict[k] == 0:
items_dict.pop(k)
results = cls.get_by_key_name(items_dict.keys())
# cls.get_by_key_name returns a quirky [ None, ] when no result can be found.
results = filter(lambda a: a != None, results)
to_add = []
to_del = []
if results:
i = 0
while i < len(results):
results[i].count += items_dict[results[i].key().name()]
items_dict.pop(results[i].key().name())
#pop out terms found in query results so the rest will be new terms.
if results[i].count < 1:
to_del.append(results[i])
results.pop(i)
else:
i += 1
"""
for i in range(len(results)):
results[i].count += items_dict[results[i].key().name()]
items_dict.pop(results[i].key().name())
#pop out terms found in query results so the rest will be new terms.
if results[i].count < 1:
to_del.append(results[i])
results.pop(i)
i -= 1
"""
# create new entries for non-existing terms
if len(items_dict) > 0:
for t in items_dict:
if items_dict[t] > 0:
# Don't create new term unless the initial count > 0
to_add.append(cls(key_name=t, count=items_dict[t]))
results += to_add
db.put(results)
# remove terms whose count < 1
if len(to_del) > 0:
db.delete(to_del)
def _queueDispatchFanIn(self,
nextEvent,
fanInPeriod=0,
retryOptions=None,
queueName=None):
""" Queues a call to .dispatch(nextEvent) in the task queue, or saves the context to the
datastore for processing by the queued .dispatch(nextEvent)
@param nextEvent: a string event
@param fanInPeriod: the period of time between fan in Tasks
@param queueName: the queue name to Queue into
@return: a taskqueue.Task instance which may or may not have been queued already
"""
assert nextEvent is not None
assert not self.get(
constants.INDEX_PARAM) # fan-in after fan-in is not allowed
assert queueName
# we pop this off here because we do not want the fan-out/continuation param as part of the
# task name, otherwise we loose the fan-in - each fan-in gets one work unit.
self.pop(constants.GEN_PARAM, None)
fork = self.pop(constants.FORK_PARAM, None)
# transfer the fan-in-group into the context (under a fixed value key) so that states beyond
# the fan-in get unique Task names
# FIXME: this will likely change once we formalize what to do post fan-in
transition = self.currentState.getTransition(nextEvent)
if self.get(transition.target.fanInGroup) is not None:
self[constants.FAN_IN_GROUP_PARAM] = self[
transition.target.fanInGroup]
taskNameBase = self.getTaskName(nextEvent, fanIn=True)
rwlock = ReadWriteLock(taskNameBase, self)
index = rwlock.currentIndex()
# (***)
#
# grab the lock - memcache.incr()
#
# on Task retry, multiple incr() calls are possible. possible ways to handle:
#
# 1. release the lock in a 'finally' clause, but then risk missing a work
# package because acquiring the read lock will succeed even though the
# work package was not written yet.
#
# 2. allow the lock to get too high. the fan-in logic attempts to wait for
# work packages across multiple-retry attempts, so this seems like the
# best option. we basically trade a bit of latency in fan-in for reliability.
#
rwlock.acquireWriteLock(index, nextEvent=nextEvent)
# insert the work package, which is simply a serialized FSMContext
workIndex = '%s-%d' % (taskNameBase, knuthHash(index))
# on retry, we want to ensure we get the same work index for this task
actualTaskName = self.__obj[constants.TASK_NAME_PARAM]
indexKeyName = 'workIndex-' + '-'.join(
[str(i) for i in [actualTaskName, fork] if i]) or None
semaphore = RunOnceSemaphore(indexKeyName, self)
# check if the workIndex changed during retry
semaphoreWritten = False
if self.__obj[constants.RETRY_COUNT_PARAM] > 0:
# see comment (A) in self._queueDispatchFanIn(...)
time.sleep(constants.DATASTORE_ASYNCRONOUS_INDEX_WRITE_WAIT_TIME)
payload = semaphore.readRunOnceSemaphore(payload=workIndex,
transactional=False)
if payload:
semaphoreWritten = True
if payload != workIndex:
self.logger.info(
"Work index changed from '%s' to '%s' on retry.",
payload, workIndex)
workIndex = payload
# update this here so it gets written down into the work package too
self[constants.INDEX_PARAM] = index
# write down two models, one actual work package, one idempotency package
keyName = '-'.join([str(i)
for i in [actualTaskName, fork] if i]) or None
work = _FantasmFanIn(context=self,
workIndex=workIndex,
key_name=keyName)
# close enough to idempotent, but could still write only one of the entities
# FIXME: could be made faster using a bulk put, but this interface is cleaner
if not semaphoreWritten:
semaphore.writeRunOnceSemaphore(payload=workIndex,
transactional=False)
# put the work item
db.put(work)
# (A) now the datastore is asynchronously writing the indices, so the work package may
# not show up in a query for a period of time. there is a corresponding time.sleep()
# in the fan-in of self.mergeJoinDispatch(...)
# release the lock - memcache.decr()
rwlock.releaseWriteLock(index)
try:
# insert a task to run in the future and process a bunch of work packages
now = time.time()
url = self.buildUrl(self.currentState, nextEvent)
params = self.buildParams(self.currentState, nextEvent)
task = Task(name='%s-%d' % (taskNameBase, index),
method=self.method,
url=url,
params=params,
eta=datetime.datetime.utcfromtimestamp(now) +
datetime.timedelta(seconds=fanInPeriod),
headers=self.headers,
retry_options=retryOptions)
self.Queue(name=queueName).add(task)
return task
except (TaskAlreadyExistsError, TombstonedTaskError):
pass # Fan-in magic
def do_task_fetch_material(username, force_top):
user = CurryUser.get_by_key_name(username)
force_top = bool(int(force_top))
if not user:
logging.error("no such user '%s'" % username)
return 'bad'
tweet_list = api.GetUserTimeline(screen_name=username,
count=config.FETCH_COUNT)
tweet = None
material_list = None
success = False
if force_top:
tweet_list = tweet_list[0:1]
else:
shuffle(tweet_list)
#
# select material
#
for tweet in tweet_list:
# check history
if not force_top and is_duplicated(tweet):
continue
text = tweet.GetText().encode('utf-8')
material_list = analyze(text, count=config.TWEET_MATERIAL_MAX)
if len(material_list) > 0:
# found material
success = True
break
if success:
# record to history
# TODO: trim history chronically
History(key_name=str(tweet.id), timestamp=datetime.now()).put()
else:
logging.info("material not found for user '%s'" % username)
return 'bad'
#
# select receivers
#
link_list = (UserLink.all().filter(
'sender = ', user).order('timestamp').fetch(limit=config.RECEIVER_MAX))
for link in link_list:
# randomize material per receiver
shuffle(material_list)
count = 1 + int(random() * len(material_list))
receive_material = material_list[:count]
taskqueue.add(queue_name='post-queue',
url='/task/post_material/%s/%s' %
(username, link.receiver.key().name()),
params={'material': receive_material})
link.timestamp = datetime.now()
logging.debug(
"sending from user '%s' to '%s' with material '%s'" %
(username, link.receiver.key().name(), repr(receive_material)))
# update timestamp
db.put(link_list)
# send to karei_bot if no receivers
if len(link_list) == 0:
shuffle(material_list)
count = 1 + int(random() * len(material_list))
receive_material = material_list[:count]
taskqueue.add(queue_name='post-queue',
url='/task/post_material/%s/%s' %
(username, config.MY_NAME),
params={'material': receive_material})
logging.debug("sending from user '%s' to '%s' with material '%s'" %
(username, config.MY_NAME, repr(receive_material)))
return 'ok'
def UpdateScheduledJobs(jobs): """Updates the scheduled jobs.""" db.put(jobs)
def test_import_notes_disabled_note_records(self):
'''Check that notes will be rejected from API import when
notes_disabled is set to be True by the record author.'''
records = []
# First prepare and import two person records
for i in range(2):
given_name = "given_name_%d" % i
family_name = "family_name_%d" % i
source_date = "2010-01-01T01:23:45Z"
record_id = "test_domain/person_%d" % i
author_name = "test_author"
author_email = "test_email"
records.append({
'given_name': given_name,
'family_name': family_name,
'person_record_id': record_id,
'source_date': source_date,
'author_name': author_name,
'author_email': author_email
})
written, skipped, total = importer.import_records(
'haiti', 'test_domain', importer.create_person, records, False,
True, None)
assert written == 2
assert len(skipped) == 0
assert total == 2
assert model.Person.all().count() == 2
# Disable comments for first person record
person = model.Person.get('haiti', 'test_domain/person_0')
assert person
person.notes_disabled = True
db.put([person])
for person in model.Person.all():
if person.person_record_id == 'test_domain/person_0':
assert person.notes_disabled == True
# Import notes
records = []
for i in range(2):
source_date = '2010-01-01T01:23:45Z'
note_id = 'test_domain/record_%d' % i
person_id = 'test_domain/person_%d' % i
records.append({
'person_record_id': person_id,
'note_record_id': note_id,
'source_date': source_date
})
written, skipped, total = importer.import_records(
'haiti', 'test_domain', importer.create_note, records, False, True,
None)
# Check that the note associted with first person record is skipped.
assert written == 1
assert len(skipped) == 1
assert skipped[0] == (
'The author has disabled new commenting on this record', {
'person_record_id': 'test_domain/person_0',
'source_date': '2010-01-01T01:23:45Z',
'note_record_id': 'test_domain/record_0',
})
assert total == 2
assert model.Note.all().count() == 1