def start_new_game(self): """ Starts a new game """ question = self.generate_question() question_template = question.question_template.get() question.times_used += 1 question_template.times_used += 1 # reset the game self.question_string = question.question self.question = question.key self.started_at = datetime.datetime.now() self.answers = [] self.background_color = random.choice(Game.GAME_COLORS) self.cached_status = None self.is_dirty = False self.players = [] self.times_played += 1 self.is_banned = False self.flagged_irrelevant = 0 self.flagged_nonsense = 0 # save the question and the game ndb.put_multi([self, question, question_template]) return self
def SetRecords(self, key, records): """Sets records for this log and put into datastore. Serializes records and save over multiple entities if necessary. Args: key: String key name of a QuickLog entity. records: List of Record object. """ # Number of bytes less than 1MB for ndb.BlobProperty. chunk_size = 1000000 serialized = pickle.dumps(records, 2) length = len(serialized) if length / chunk_size > _MAX_NUM_PARTS: logging.error('Data too large to save.') return None log_parts = [] for i in xrange(0, length, chunk_size): # +1 to start entitiy key at 1. part_id = i // chunk_size + 1 part_value = serialized[i:i+chunk_size] parent_key = ndb.Key('QuickLog', key) log_part = QuickLogPart(id=part_id, parent=parent_key, value=part_value) log_parts.append(log_part) self.size = len(log_parts) ndb.put_multi(log_parts + [self])
def test_append_output_max_chunk(self): # This test case is very slow (1m25s locally) if running with the default # values, so scale it down a bit which results in ~2.5s. self.mock( task_result.TaskOutput, 'PUT_MAX_CONTENT', task_result.TaskOutput.PUT_MAX_CONTENT / 8) self.mock( task_result.TaskOutput, 'PUT_MAX_CHUNKS', task_result.TaskOutput.PUT_MAX_CHUNKS / 8) self.assertFalse( task_result.TaskOutput.PUT_MAX_CONTENT % task_result.TaskOutput.CHUNK_SIZE) calls = [] self.mock(logging, 'error', lambda *args: calls.append(args)) max_chunk = 'x' * task_result.TaskOutput.PUT_MAX_CONTENT entities = self.run_result.append_output(0, max_chunk, 0) self.assertEqual(task_result.TaskOutput.PUT_MAX_CHUNKS, len(entities)) ndb.put_multi(entities) self.assertEqual([], calls) # Try with PUT_MAX_CONTENT + 1 bytes, so the last byte is discarded. entities = self.run_result.append_output(1, max_chunk + 'x', 0) self.assertEqual(task_result.TaskOutput.PUT_MAX_CHUNKS, len(entities)) ndb.put_multi(entities) self.assertEqual(1, len(calls)) self.assertTrue(calls[0][0].startswith('Dropping '), calls[0][0]) self.assertEqual(1, calls[0][1])
def flag(self, reason): """ Add as a bad question 1 - Nonsense 2 - Irrelevant """ if reason == 1: self.flagged_nonsense += 1 else: # problem_type = 2 self.flagged_irrelevant += 1 percent_flagged = float(len(self.players)) / self.times_flagged logging.error("%f percent flagged" % (percent_flagged)) if self.times_flagged > 1 and percent_flagged > 0.34 and \ self.duration() > 1.5 and self.duration() < 10: # ban the question question = self.question.get() question.is_banned = True self.is_banned = True ndb.put_multi([self, question]) logging.info("Question banned") return True return False
def test_append_output_partial(self): ndb.put_multi(self.run_result.append_output(0, 'Foo', 10)) expected_output = '\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00Foo' self.assertEqual( expected_output, self.run_result.get_command_output_async(0).get_result()) self.assertTaskOutputChunk([{'chunk': expected_output, 'gaps': [0, 10]}])
def get_tweets(): disease_terms = [ 'dengue', 'gripe', 'gripado', 'gripada', 'h1n1', 'tuberculose', u'cólera', 'colera', u'malária', 'malaria' ] for term in disease_terms: tweets = twitter_api.search(q=term, rpp=100) documents = [ Document(id=tweet.id_str, text=tweet.text) for tweet in tweets ] ndb.put_multi(documents)
def send_contact_mail(self): contacts = Contact.query(Contact.sent != True) count = contacts.count() if count: datetime_handler = lambda obj: obj.isoformat() \ if isinstance(obj, datetime) else None message = json.dumps( [contact.to_dict() for contact in contacts], default=datetime_handler, indent=4 ) mail.send_mail( sender="Getmewrite.com Support <*****@*****.**>", to="Mona <*****@*****.**>", subject="You have {0} new contact(s)".format(count), body=message ) self.response.write(message) logger.info( 'Send daily mail success, {0} new contacts'.format(count)) put_list = [] for contact in contacts: contact.sent = True put_list.append(contact) ndb.put_multi(put_list)
def test_set_from_run_result_two_tries(self): request = task_request.make_request(_gen_request(), True) result_summary = task_result.new_result_summary(request) run_result_1 = task_result.new_run_result( request, 1, 'localhost', 'abc', {}) run_result_2 = task_result.new_run_result( request, 2, 'localhost', 'abc', {}) self.assertTrue(result_summary.need_update_from_run_result(run_result_1)) run_result_2.modified_ts = utils.utcnow() result_summary.modified_ts = utils.utcnow() ndb.transaction(lambda: ndb.put_multi((result_summary, run_result_2))) self.assertTrue(result_summary.need_update_from_run_result(run_result_1)) run_result_1.modified_ts = utils.utcnow() result_summary.set_from_run_result(run_result_1, request) ndb.transaction(lambda: ndb.put_multi((result_summary, run_result_1))) result_summary = result_summary.key.get() self.assertFalse(result_summary.need_update_from_run_result(run_result_1)) self.assertTrue(result_summary.need_update_from_run_result(run_result_2)) run_result_2.modified_ts = utils.utcnow() result_summary.set_from_run_result(run_result_2, request) ndb.transaction(lambda: ndb.put_multi((result_summary, run_result_2))) result_summary = result_summary.key.get() self.assertEqual(2, result_summary.try_number) self.assertFalse(result_summary.need_update_from_run_result(run_result_1))
def refreshData(): try: spider = citySpider() city_list = spider.parse() """First empty the data-store""" city_query = cityDB.query() entity_list = city_query.fetch() key_list = [] for entity in entity_list: key_list.append(entity.key) ndb.delete_multi(key_list) """Store fresh cities""" entity_list = [] for city in city_list: city_name = "" for i in xrange(len(city[0])): # replcaing spaces with '-'(dash) if city[0][i] == ' ': city_name += '-' else: city_name += city[0][i] city_name = city_name.lower() mykey = ndb.Key('cityDB',city_name) if cityDB.get_by_id(city_name) is None: entity = cityDB(key = mykey, alt_name = city[1], timestamp = dt.datetime(2000,1,1,0,0,0,0)) else: raise Exception("Assertion Fail. Two cities with same name encountered") entity_list.append(entity) ndb.put_multi(entity_list) except Exception as exp: raise exp
def _put_annotations_batch(annotation_data): try: annotation_objects = [] qry = BookRecord.query( BookRecord.item_id_array.IN(annotation_data.keys()) ) keys = qry.fetch(MAX_OBJECTS_PER_BATCH, keys_only=True) for key in keys: first_item_id = key.string_id().split('|')[0] if first_item_id not in annotation_data: continue short_text = annotation_data[first_item_id][0] long_text = annotation_data[first_item_id][1] anno_key = ndb.Key(BookAnnotation, key.string_id()) annotation = BookAnnotation(key=anno_key, short=short_text, long=long_text) annotation_objects.append(annotation) ndb.put_multi(annotation_objects) logging.info("{} annotations put into datastore" .format(len(annotation_objects))) except Exception as e: logging.error(e) raise deferred.PermanentTaskFailure()
def testGetGraphJsonError(self): self._AddTestColumns(start_rev=15000, end_rev=15015) rows = graph_data.Row.query( graph_data.Row.parent_test == ndb.Key( 'TestMetadata', 'ChromiumGPU/win7/dromaeo/dom')) for row in rows: row.error = 1 + ((row.revision - 15000) * 0.25) ndb.put_multi(rows) flot_json_str = graph_json.GetGraphJson( { 'ChromiumGPU/win7/dromaeo/dom': [], }) flot = json.loads(flot_json_str) self.assertEqual(1, len(flot['error_bars'].keys())) rev = 0 for col_dom, col_top, col_bottom in zip( flot['data']['0']['data'], flot['error_bars']['0'][1]['data'], flot['error_bars']['0'][0]['data']): error = 1 + (rev * 0.25) self.assertEqual(rev + 15000, col_top[0]) self.assertEqual(col_dom[1] + error, col_top[1]) self.assertEqual(rev + 15000, col_bottom[0]) self.assertEqual(col_dom[1] - error, col_bottom[1]) rev += 3
def _AddLongTestColumns(self, start_rev=15000, end_rev=16500, step=3): """Adds test data with long nested sub test to the mock datastore. Args: start_rev: Starting revision number. end_rev: Ending revision number. step: Difference between adjacent revisions. """ master = graph_data.Master(id='master') master.put() bot = graph_data.Bot(id='bot', parent=master.key) bot.put() test = graph_data.TestMetadata(id='master/bot/suite') test.put() rows = [] path = 'master/bot/suite' for sub_name in ['sub1', 'sub2', 'sub3', 'sub4', 'sub5']: path = '%s/%s' % (path, sub_name) test = graph_data.TestMetadata(id=path, improvement_direction=anomaly.UP, has_rows=True) test.put() test_container_key = utils.GetTestContainerKey(test.key) for i in range(start_rev, end_rev, step): row = graph_data.Row( parent=test_container_key, id=i, value=float(i * 2), r_webkit=int(i * 0.25), a_str='some_string', buildnumber=i - start_rev, a_tracing_uri='http://trace/%d' % i, a_trace_rerun_options={'foo': '--foo'}) rows.append(row) ndb.put_multi(rows)
def get(self): cursorString = self.request.get('cursor') cursor = None if cursorString != 'None': cursor = Cursor(urlsafe=cursorString) listVectors, next_cursor, more = Vector.query().fetch_page(BATCH_SIZE, start_cursor=cursor) to_put = [] for vector in listVectors: if any([ (vector.server == 'OGC' and vector.dataset == 'BusStops' and vector.name == 'GetCapabilities' and vector.httpMethod == 'GET' and vector.returnType == 'XML'), (vector.server == 'OGC' and vector.dataset == 'BusStops' and vector.name == 'AttributeFilter' and vector.httpMethod == 'GET' and vector.returnType == 'JSON'), (vector.server == 'OGC' and vector.dataset == 'BusStops' and vector.name == 'GetCapabilities' and vector.httpMethod == 'POST' and vector.returnType == 'XML'), (vector.server == 'OGC' and vector.dataset == 'Topo' and vector.name == 'Big' and vector.httpMethod == 'GET' and vector.returnType == 'Image'), (vector.server == 'OGC' and vector.dataset == 'Topo' and vector.name == 'Small' and vector.httpMethod == 'GET' and vector.returnType == 'Image'), (vector.server == 'GME' and vector.dataset == 'BusStops' and vector.name == 'Small' and vector.httpMethod == 'GET' and vector.returnType == 'JSON'), (vector.server == 'GME' and vector.dataset == 'AerialPhoto' and vector.name == 'WMSGetCapabilities' and vector.httpMethod == 'GET' and vector.returnType == 'XML'), (vector.server == 'GME' and vector.dataset == 'AerialPhoto' and vector.name == 'WMTSGetCapabilities' and vector.httpMethod == 'GET' and vector.returnType == 'XML') ]): vector.referenceCheckValid = False print 'Changed flag! False!' to_put.append(vector) else: vector.referenceCheckValid = True print 'Changed flag! True!' to_put.append(vector) if to_put: ndb.put_multi(to_put) if more: print next_cursor.urlsafe() taskqueue.add(url='/updateschemaworker', method='GET', params={'cursor':next_cursor.urlsafe()})
def init(): """ dbの初期状態を作成する.すでに初期化されている場合は異常を返す 変数は取らない 返却型はBooleanで初期化成功はTrue、失敗はFalse """ key = ndb.Key('PokemonList', u'フシギダネ') dao = key.get() if dao == None: daoList = [] for normalOrder, (pokemon, gengarOrder, nicoTime) in enumerate(zip(POKEMON_LIST, GENGARS_ORDER, NICO_TIMES)): dao = PokemonList(id=pokemon) dao.normalOrder = normalOrder dao.gengarOrder = gengarOrder dao.pokemon = pokemon dao.clefableFlag = False dao.gengarFlag = False dao.nicoFlag = True dao.nicoTime = nicoTime daoList.append(dao) ndb.put_multi(daoList) logging.info(u"PokemonListの初期化に成功しました") return True else: logging.warning(u"すでにPokemonListは初期化されています") return False
def get_or_create_multi(cls, exploration_id, rule_data): """Gets or creates entities for the given rules. Args: exploration_id: the exploration id rule_data: a list of dicts, each with the following keys: (state_name, rule_str). """ # TODO(sll): Use a hash instead to disambiguate. entity_ids = ['.'.join([ exploration_id, datum['state_name'], _OLD_SUBMIT_HANDLER_NAME, datum['rule_str'] ])[:490] for datum in rule_data] entity_keys = [cls._get_entity_key(exploration_id, entity_id) for entity_id in entity_ids] entities = ndb.get_multi(entity_keys) entities_to_put = [] for ind, entity in enumerate(entities): if entity is None: new_entity = cls(id=entity_ids[ind], answers={}) entities_to_put.append(new_entity) entities[ind] = new_entity ndb.put_multi(entities_to_put) return entities
def post(self): videoBlobInfo = self.get_uploads('video') screenshotBlobInfo = self.get_uploads('screenshot') title = self.request.get("title") description = self.request.get("description") if len( videoBlobInfo ) != 1 or len( screenshotBlobInfo ) != 1 or not title or not description: return self.uploadError( 400 ) if not users.is_current_user_admin(): return self.uploadError( 401 ) uploadRoot = self.request.host_url + BLOB_ROOT videoBlobKey = str(videoBlobInfo[0].key()) videoBlobUri = uploadRoot + videoBlobKey + "/" + videoBlobInfo[0].filename screenshotBlobKey = str(screenshotBlobInfo[0].key()) screenshotBlobUri = uploadRoot + screenshotBlobKey + "/" + screenshotBlobInfo[0].filename userId = users.get_current_user().user_id() userEntity = User.getById( userId ) videoEntity = Video(id=videoBlobKey, videoUri=videoBlobUri, screenshotUri=screenshotBlobUri, uploadUser=userId, title=title, description=description, vector=[]) ndb.put_multi([userEntity, videoEntity]) self.response.headers['Content-Type'] = 'application/json' self.response.out.write( json.dumps({ 'id' : videoBlobKey }) )
def release_lease(lease_key): """Releases a lease on a machine. Args: lease_key: ndb.Key for a models.LeaseRequest entity. """ lease = lease_key.get() if not lease: logging.warning('LeaseRequest not found: %s', lease_key) return if not lease.released: logging.warning('LeaseRequest not released:\n%s', lease) return lease.released = False if not lease.machine_id: logging.warning('LeaseRequest has no associated machine:\n%s', lease) lease.put() return machine = ndb.Key(models.CatalogMachineEntry, lease.machine_id).get() if not machine: logging.error('LeaseRequest has non-existent machine leased:\n%s', lease) lease.put() return # Just expire the lease now and let MachineReclamationProcessor handle it. logging.info('Expiring LeaseRequest:\n%s', lease) now = utils.utcnow() lease.response.lease_expiration_ts = utils.datetime_to_timestamp( now) / 1000 / 1000 machine.lease_expiration_ts = now ndb.put_multi([lease, machine])
def _AddRowsFromIterable(container_key, row_ids): """Adds a set of Rows given an iterable of ID numbers.""" rows = [] for int_id in sorted(row_ids): rows.append(graph_data.Row(id=int_id, parent=container_key, value=int_id)) ndb.put_multi(rows) return rows
def txn(): first_batch_call = not all([hasattr(self, "_nodes_to_put"), hasattr(self, "_indices_to_put"), hasattr(self, "_identifier_cache"), hasattr(self, "_keys_to_delete")]) if first_batch_call: self._nodes_to_put = dict() self._indices_to_put = dict() self._identifier_cache = dict() self._keys_to_delete = set() try: results = func() if first_batch_call and any([self._nodes_to_put, self._indices_to_put, self._keys_to_delete]): futures = ndb.delete_multi_async(self._keys_to_delete) ndb.put_multi(chain(self._nodes_to_put.itervalues(), self._indices_to_put.itervalues())) [future.get_result() for future in futures] finally: if first_batch_call: del self._nodes_to_put del self._indices_to_put del self._identifier_cache del self._keys_to_delete return results
def commit(self): assert not self.committed puts = [c.entity.make_historical_copy(c.deletion, c.comment) for c in self.changes] ndb.put_multi(puts + [self.replication_state]) for cb in _commit_callbacks: cb(self.replication_state.auth_db_rev) self.committed = True
def test_append_output_overwrite(self): # Overwrite previously written data. ndb.put_multi(self.run_result.append_output(0, 'FooBar', 0)) ndb.put_multi(self.run_result.append_output(0, 'X', 3)) self.assertEqual( 'FooXar', self.run_result.get_command_output_async(0).get_result()) self.assertTaskOutputChunk([{'chunk': 'FooXar', 'gaps': []}])
def update_content_details(id): to_put = [] content = get_content_or_404(id) ref_path = i18n_path(request.forms.get('back', content.path)) if not content.is_editable: response.flash(_('Voting is disabled for content that is being ' 'broadcast')) redirect(ref_path) vote = request.forms.get('vote') if vote not in ['up', 'down']: response.flash(_('There was a problem with the request. Please try ' 'again later.')) redirect(ref_path) if vote == 'up': content.upvotes += 1 to_put.append(Event.create(Event.UPVOTE, content.key)) elif vote == 'down': content.downvotes += 1 to_put.append(Event.create(Event.DOWNVOTE, content.key)) to_put.append(content) ndb.put_multi(to_put) redirect(ref_path)
def addN(self, quads): # TODO: Handle splitting large graphs into two entities # Note: quads is a generator, not a list. It cannot be traversed twice. # Step 1: Collect the triples into the Graphs reflecting the GraphShards they will be added to. new_shard_dict = defaultdict(Graph) for (s, p, o, _) in quads: # Last component ignored as this Store is not context_aware subject_shard = choice(self.keys_for(self._ID, s, 0)) new_shard_dict[subject_shard].add((s, p, o)) predicate_shard = choice(self.keys_for(self._ID, p, 1)) new_shard_dict[predicate_shard].add((s, p, o)) keys = list(new_shard_dict.keys()) # Step 2: Load all existing, corresponding GraphShards keys_models = zip(keys, ndb.get_multi(keys)) # TODO: Use async get # Step 3: Update or create GraphShards with the added triples updated = list() for index in range(len(keys_models)): (key, model) = keys_models[index] if model is None: model = GraphShard(key=key, graph_ID=self._ID, graph_n3=new_shard_dict[key].serialize(format="n3")) else: new_shard_dict[key].parse(data=model.graph_n3, format="n3") model.graph_n3 = new_shard_dict[key].serialize(format="n3") updated.append(model) # Step 4: Invalidate and store all created/updated GraphShards if len(updated) > 0: GraphShard.invalidate(updated) ndb.put_multi(updated)
def migrate(dry_run=False): to_put = [ProfileAppEmailMapping(key=ProfileAppEmailMapping.create_key(p.app_email), username=p.username) for p in Profile.list_with_app_user()] if dry_run: return len(to_put), to_put for parts in chunks(to_put, 200): ndb.put_multi(parts)
def update_content_details(id): content = get_content_or_404(id) if not content.is_editable: # Translators, shown when content is not editable (it's on air, etc) response.flash(_('This content is not editable')) redirect(i18n_path(content.path)) errors = {} title = request.forms.getunicode('title', '').strip() license = request.forms.get('license') or None if not content.title and not title: errors['title'] = _('Title cannot be blank') if license and license not in Content.LICENSE_CHOICES: errors['license'] = _('Please select a license from provided choices') if not errors: to_put = [] if title and content.title != title: content.title = title to_put.append(Event.create(Event.TITLE, content.key)) if license and content.license != license: content.license = license to_put.append(Event.create(Event.LICENSE, content.key)) if to_put: # If we have events in to_put list, we also need to put the content to_put.append(content) ndb.put_multi(to_put) response.flash(_('Content has been updated')) redirect(content.path) return dict(vals=request.forms, errors=erorrs, content=content)
def post(self): self._require_admin() additions = json.loads(self.request.get("youtube_additions_json")) match_keys, youtube_videos = zip(*additions["videos"]) matches = ndb.get_multi([ndb.Key(Match, match_key) for match_key in match_keys]) matches_to_put = [] results = {"existing": [], "bad_match": [], "added": []} for (match, match_key, youtube_video) in zip(matches, match_keys, youtube_videos): if match: if youtube_video not in match.youtube_videos: match.youtube_videos.append(youtube_video) matches_to_put.append(match) results["added"].append(match_key) else: results["existing"].append(match_key) else: results["bad_match"].append(match_key) ndb.put_multi(matches_to_put) # TODO use Manipulators -gregmarra 20121006 self.template_values.update({ "results": results, }) path = os.path.join(os.path.dirname(__file__), '../../templates/admin/videos_add.html') self.response.out.write(template.render(path, self.template_values))
def start_draft(self, draft): draft.in_setup = False draft.in_progress = True #shuffle user_keys to create random order random.shuffle(draft.user_keys) #create the individual drafter items position = 0 drafter_entities = [] for user_key in draft.user_keys: drafter_entities.append(Drafter.create(user_key=user_key, draft_key=draft.key, position=position)) position += 1 draft.drafter_keys = ndb.put_multi(drafter_entities) # logging.error('424: draft') # logging.error(draft) pack_entities = [] #get boosters for all the packs for pack_code in draft.pack_codes: packs = SetUtil().generate_boosters( num=draft.num_drafters, set_code=pack_code) random.shuffle(packs) for pack in packs: pack_entities.append(Pack.create(draft_key=draft.key, cards=pack)) draft.unopened_pack_keys = ndb.put_multi(pack_entities) draft.put() # time.sleep(5) # logging.error('440: draft') # logging.error(draft) self.next_pack(draft)
def post(self, namespace, timestamp): digests = [] now = utils.timestamp_to_datetime(long(timestamp)) expiration = config.settings().default_expiration try: digests = payload_to_hashes(self, namespace) # Requests all the entities at once. futures = ndb.get_multi_async( model.entry_key(namespace, binascii.hexlify(d)) for d in digests) to_save = [] while futures: # Return opportunistically the first entity that can be retrieved. future = ndb.Future.wait_any(futures) futures.remove(future) item = future.get_result() if item and item.next_tag_ts < now: # Update the timestamp. Add a bit of pseudo randomness. item.expiration_ts, item.next_tag_ts = model.expiration_jitter( now, expiration) to_save.append(item) if to_save: ndb.put_multi(to_save) logging.info( 'Timestamped %d entries out of %s', len(to_save), len(digests)) except Exception as e: logging.error('Failed to stamp entries: %s\n%d entries', e, len(digests)) raise
def post(self): self.results = defaultdict(list) self.updated = [] user_key = users.user_key(self.user) request = json.loads(self.request.body) for stream_updates in request: stream = streams.stream_key( stream_updates['name'], user_key = user_key, create=False, ).get() if not stream: self.results['failed'].append(stream_updates['name']) continue if stream_updates.get('publishers'): self.update_publishers(stream, stream_updates['publishers']) if stream_updates.get('volumes'): self.update_volumes(stream, stream_updates['volumes']) if stream_updates.get('issues'): self.update_issues(stream, stream_updates['issues']) if self.updated: ndb.put_multi(self.updated) status = 200 message = '%d stream changes' % len(self.updated) else: status = 203 message = 'no changes' self.response.write(json.dumps({ 'status': status, 'message': message, 'results': self.results, }))
def _AddUserRecordsToDB(self, users_to_add): """Helper to perform ndb put() with error handling. Args: users_to_add: List of user (DomainUserToCheckModel) entities to add. """ ndb.put_multi(users_to_add)
def init_zoneinfo(): """ Add each zone info to the datastore. This will overwrite existing zones. This must be called before the AppengineTimezoneLoader will work. """ import os, logging from zipfile import ZipFile zoneobjs = [] zoneinfo_path = os.path.abspath(os.path.join(os.path.dirname(__file__), 'zoneinfo.zip')) with ZipFile(zoneinfo_path) as zf: for zfi in zf.filelist: key = ndb.Key('Zoneinfo', zfi.filename, namespace=NDB_NAMESPACE) zobj = Zoneinfo(key=key, data=zf.read(zfi)) zoneobjs.append(zobj) logging.info("Adding %d timezones to the pytz-appengine database" % len(zoneobjs) ) ndb.put_multi(zoneobjs)
def _create_token_value_agreement_if_needed(profile_key): profile = profile_key.get() # type: TffProfile investments = [ i for i in InvestmentAgreement.list_by_user(profile.app_user) if PaymentInfo.HAS_MULTIPLIED_TOKENS not in i.payment_info and i.creation_time <= FF_ENDED_TIMESTAMP ] statuses = [ InvestmentAgreement.STATUS_PAID, InvestmentAgreement.STATUS_SIGNED ] canceled_or_started_investments = [ i for i in investments if i.status not in statuses ] to_put, token_count = multiply_tokens_for_agreements( canceled_or_started_investments) azzert(token_count == 0, 'Expected token_count to be 0') logging.info('Updated %s agreements for user %s', len(to_put), profile.username) if to_put: ndb.put_multi(to_put) has_document = any(d.type == DocumentType.TOKEN_VALUE_ADDENDUM.value for d in Document.list_by_username(profile.username)) if any(i.status in statuses for i in investments) and not has_document: create_token_value_agreement(profile.username)
def testGetGraphJsonError(self): self._AddTestColumns(start_rev=15000, end_rev=15015) rows = graph_data.Row.query(graph_data.Row.parent_test == ndb.Key( 'Master', 'ChromiumGPU', 'Bot', 'win7', 'Test', 'dromaeo', 'Test', 'dom')).fetch() for row in rows: row.error = 1 + ((row.revision - 15000) * 0.25) ndb.put_multi(rows) flot_json_str = graph_json.GetGraphJson({ 'ChromiumGPU/win7/dromaeo/dom': [], }) flot = json.loads(flot_json_str) self.assertEqual(1, len(flot['error_bars'].keys())) rev = 0 for col_dom, col_top, col_bottom in zip( flot['data']['0']['data'], flot['error_bars']['0'][1]['data'], flot['error_bars']['0'][0]['data']): error = 1 + (rev * 0.25) self.assertEqual(rev + 15000, col_top[0]) self.assertEqual(col_dom[1] + error, col_top[1]) self.assertEqual(rev + 15000, col_bottom[0]) self.assertEqual(col_dom[1] - error, col_bottom[1]) rev += 3
def get(self): teamsToPut = [] url = "http://stats.nba.com/stats/leaguedashteamstats?Conference=&DateFrom=&DateTo=&Division=&GameScope=&GameSegment=&LastNGames=0&LeagueID=00&Location=&MeasureType=Advanced&Month=0&OpponentTeamID=0&Outcome=&PORound=0&PaceAdjust=N&PerMode=Totals&Period=0&PlayerExperience=&PlayerPosition=&PlusMinus=N&Rank=N&Season=2016-17&SeasonSegment=&SeasonType=Regular+Season&ShotClockRange=&StarterBench=&TeamID=0&VsConference=&VsDivision=" response = urlfetch.fetch(url) jsonData = json.loads(response.content) teams = jsonData['resultSets'][0]['rowSet'] statHeaders = jsonData['resultSets'][0]['headers'] for i in range(len(teams)): teamData = {} for j in range(len(teams[i])): teamData[statHeaders[j]] = teams[i][j] newTeam = Team( teamId = teams[i][0], name = teams[i][1], data = teamData, id = teams[i][0] ) teamsToPut.append(newTeam) ndb.put_multi(teamsToPut)
def post(self): if self.request.get('auth') == 'bd2c952a4b2febc39b81c967cd8556cd': response = {} response['cursor'] = '' query = APIData.query() query = query.filter(APIData.indexed_data == 'TYPE->PROJECT') query = query.filter(APIData.indexed_data == 'COA->0') n = 20 if self.request.get('n'): n = int(self.request.get('n')) if self.GET("cursor"): curs = Cursor(urlsafe=self.request.get("cursor")) data, cursor, more = query.fetch_page(n, start_cursor=curs) else: data, cursor, more = query.fetch_page(n) things_to_put = [] for d in data: d.additional_data['coa'] = '0' things_to_put.append(d) if things_to_put: ndb.put_multi(things_to_put) response['cursor'] = cursor.urlsafe() self.response.headers['Content-Type'] = 'application/json' self.response.write(json.dumps(response))
def _AddLongTestColumns(self, start_rev=15000, end_rev=16500, step=3): """Adds test data with long nested sub test to the mock datastore. Args: start_rev: Starting revision number. end_rev: Ending revision number. step: Difference between adjacent revisions. """ master = graph_data.Master(id='master') master.put() bot = graph_data.Bot(id='bot', parent=master.key) bot.put() test = graph_data.TestMetadata(id='master/bot/suite') test.UpdateSheriff() test.put() rows = [] path = 'master/bot/suite' for sub_name in ['sub1', 'sub2', 'sub3', 'sub4', 'sub5']: path = '%s/%s' % (path, sub_name) test = graph_data.TestMetadata(id=path, improvement_direction=anomaly.UP, has_rows=True) test.UpdateSheriff() test.put() test_container_key = utils.GetTestContainerKey(test.key) for i in range(start_rev, end_rev, step): row = graph_data.Row(parent=test_container_key, id=i, value=float(i * 2), r_webkit=int(i * 0.25), a_str='some_string', buildnumber=i - start_rev, a_tracing_uri='http://trace/%d' % i) rows.append(row) ndb.put_multi(rows)
def recalculate_goat_index(sport): ndb.delete_multi(UserGoatIndex.query().filter( UserGoatIndex.sport == sport).fetch(keys_only=True)) q = Pick.query().filter(Pick.sport == sport) # sport == "nba" picks = q.fetch() goat_indexes = [] for pick in picks: #get_or_insert user goat index user_id = pick.user_id event = pick.event.get() if event.start_time < datetime.datetime.utcnow(): goat_index = UserGoatIndex.get_or_insert("{}{}".format( sport, user_id), user_id=user_id, sport=sport, num_pick=0, num_point=0, num_correct=0) outcome = event.outcome goat_index.num_pick = goat_index.num_pick + 1 if outcome.winner == pick.pick: goat_index.num_correct = goat_index.num_correct + 1 goat_indexes.append(goat_index) ndb.put_multi(goat_indexes)
def create_project_cohort(self, cohort_date=datetime.datetime.today()): program = Program.get_config(self.program_label) liaison = User.create(email='*****@*****.**') org = Organization.create(name="Org Foo", liaison_id=liaison.uid) liaison.owned_organizations.append(org.uid) project = Project.create(organization_id=org.uid, program_label=self.program_label) liaison.put() org.put() project.put() pc = ProjectCohort.create( project_id=project.uid, organization_id=org.uid, program_label=self.program_label, cohort_label=self.cohort_label, liaison_id=liaison.uid, ) pc.put() surveys = Survey.create_for_project_cohort(program['surveys'], pc) ndb.put_multi(surveys) return liaison, org, project, pc, surveys
def put_multi(models, update_last_updated_time=True): """Stores a sequence of Model instances. Args: models: list(datastore_services.Model). A list of Model instances. update_last_updated_time: bool. Whether to update the last_updated field of the entities. Returns: list(str). A list with the stored keys. """ # TODO(#10863): Stop passing in update_last_updated_time through these # top-level functions. return ndb.put_multi( models, update_last_updated_time=update_last_updated_time)
def dispatch_request(self): form = TokenForm() if form.validate_on_submit(): token = uuid4().hex[:6].upper() phone = phonenumbers.format_number( phonenumbers.parse(form.user_phone.data, region='US'), phonenumbers.PhoneNumberFormat.E164) registered_users = UserModel.query(UserModel.user_phone == phone) updated_users = [] for registered_user in registered_users: registered_user.user_token = token updated_users.append(registered_user) if updated_users: ndb.put_multi(updated_users) url = request.url_root.rstrip('/') + url_for('validate', user_token=token) client = TwilioRestClient(TWILIO_SID, TWILIO_TOKEN) message = client.messages.create(body="Jeff's Xmas Tracker\n\n" "Token: %s\n" "or\n" "Click: %s" % (token, url), to=phone, from_="+15153052239") flash( u"Token sent to %s. Follow link or type in token above." % phone, 'success') else: flash( u"The phone number %s isn't registered to any users." % phone, 'warning') return render_template('validate_token.html', form=form)
def GetFailureKeysToAnalyzeTestFailures(self, failure_entities): """Gets failures that'll actually be analyzed in the analysis. Groups failures by suite, picks one failure per group and links other failures in group to it. Note because of the lack of test level failure info, such in-build grouping could cause false positives, but we still decide to do it in consideration of saving resources and speeding up analysis. """ suite_to_failure_map = defaultdict(list) for failure in failure_entities: properties = failure.properties or {} if not properties.get('needs_bisection', True): # Should not include the failure if it doesn't need bisection. continue suite_to_failure_map[properties.get('suite')].append(failure) analyzing_failure_keys = [] failures_to_update = [] for same_suite_failures in suite_to_failure_map.itervalues(): sample_failure_key = same_suite_failures[0].key analyzing_failure_keys.append(sample_failure_key) if len(same_suite_failures) == 1: continue for i in xrange(1, len(same_suite_failures)): # Merges the rest of failures into the sample failure. failure = same_suite_failures[i] failure.merged_failure_key = sample_failure_key failures_to_update.append(failure) if failures_to_update: ndb.put_multi(failures_to_update) return analyzing_failure_keys
def post(self): self.response.headers.add_header('Access-Control-Allow-Origin', '*') data = self.request.body email = getEmail(json.loads(self.request.body)) if (email == None): msg = "invalid user" self.response.write(json.dumps(msg)) return deleteCourses(email) j = json.loads(data) courses = j['courselist'] ents = [] l = len(courses) for i in range(0, l): print courses[i] crs = Course(email=email, name=courses[i]['name'], description=courses[i]['description']) ents.append(crs) ndb.put_multi(ents) self.response.headers['Content-Type'] = 'application/json' self.response.write(data)
def touch_auth_db(): # Recheck under transaction. if key.get() is not None: return to_process = [] # Start slow queries in parallel. groups_future = model.AuthGroup.query( ancestor=model.root_key()).fetch_async() whitelists_future = model.AuthIPWhitelist.query( ancestor=model.root_key()).fetch_async() # Singleton entities. to_process.append(model.root_key().get()) to_process.append(model.ip_whitelist_assignments_key().get()) # Finish queries. to_process.extend(groups_future.get_result()) to_process.extend(whitelists_future.get_result()) # Update auth_db_rev properties, make *History entities. Keep modified_by # and modified_ts as they were. to_put = [] for ent in to_process: if not ent: continue ent.record_revision( modified_by=ent.modified_by, modified_ts=ent.modified_ts, comment='Initial snapshot') to_put.append(ent) # Store changes, update the marker to make sure this won't run again. ndb.put_multi(to_put) auth_db_rev = model.replicate_auth_db() _AuthDBSnapshotMarker(key=key, auth_db_rev=auth_db_rev).put()
def post(self): register_name = self.request.get('register_name', DEFAULT_REGISTER_NAME) all_trades = ClerkshipTrade.query( ancestor=register_key(register_name)).fetch() # zero out past matches for trade in all_trades: trade.match_to_desired = None trade.match_to_current = None ndb.put_multi(all_trades) # Get request and update trades to match results = json.loads(self.request.body) for key, matches in results['data'].iteritems(): trade = ndb.Key(urlsafe=key).get() to_current_key = ndb.Key(urlsafe=matches['match_to_current']) to_desired_key = ndb.Key(urlsafe=matches['match_to_desired']) trade.match_to_current = to_current_key trade.match_to_desired = to_desired_key trade.put() self.response.write('Save successful')
def action(self, d): ''' ''' action = self.request.get('action') res = {} if action == 'archive_complete': recent = Task.Recent(self.user, limit=20) to_archive = [] for t in recent: if not t.archived and t.is_done(): t.archive() to_archive.append(t) if to_archive: ndb.put_multi(to_archive) res['archived_ids'] = [t.key.id() for t in to_archive] self.message = "Archived %d %s" % ( len(to_archive), tools.pluralize('task', count=len(to_archive))) else: self.message = "No completed tasks to archive" self.success = True else: self.message = "Unknown action" self.set_response(res)
def importStories(r, limit=400, skip=0, overwrite=False): storyList = Story.Query.all().order_by('title').limit(limit).skip(skip) storyCount = 0 wordCount = 0 r.out.write('got stories from parse, overwrite=' + str(overwrite) + '<br/>') targetList = [] for s in storyList: storyCount += 1 r.out.write(str(storyCount) + '. ') story = processStory(s, r, overwrite) if story: targetList.append(story) wordCount += story.wordCount if (storyCount > 0): source = db.StorySource(title='Nature', storyCount=storyCount, wordCount=wordCount) targetList.append(source) ndb.put_multi(targetList) r.out.write('end of importStories<br/>')
def update_index(cls, title, old_data, new_data): old_pairs = cls.data_as_pairs(old_data) new_pairs = cls.data_as_pairs(new_data) deletes = old_pairs.difference(new_pairs) inserts = new_pairs.difference(old_pairs) # delete queries = [cls.query(cls.title == title, cls.name == name, cls.value == unicode(v.pvalue if isinstance(v, schema.Property) else v)) for name, v in deletes if not isinstance(v, schema.Property) or v.should_index()] entities = reduce(lambda a, b: a + b, [q.fetch() for q in queries], []) keys = [e.key for e in entities] if len(keys) > 0: ndb.delete_multi(keys) # insert entities = [cls(title=title, name=name, value=unicode(v.pvalue if isinstance(v, schema.Property) else v)) for name, v in inserts if not isinstance(v, schema.Property) or v.should_index()] if len(entities) > 0: ndb.put_multi(entities)
def get(self): q = taskqueue.Queue('views') while True: try: tasks = q.lease_tasks(60, 1000) if not tasks: return tallies = {} for t in tasks: tallies[t.payload] = tallies.get(t.payload, 0) + 1 objects = ndb.get_multi( [ndb.Key(BlogPost, int(k)) for k in tallies]) for object in objects: object.views += tallies[str(object.key.id())] ndb.put_multi(objects) q.delete_tasks(tasks) except google.appengine.api.taskqueue.TransientError: print("google.appengine.api.taskqueue.TransientError") except google.appengine.runtime.apiproxy_errors.DeadlineExceededError: print( "google.appengine.runtime.apiproxy_errors.DeadlineExceededError" )
def accept_invite(member_id, token): member = TenantMembership.from_urlsafe(member_id) if not member: time.sleep(config.security_wait) return flask.render_template('no-invite-token.html'), 404 assert isinstance(member, TenantMembership) if member.is_token_valid(token): # Invite accepted if g.current_account: member.user = g.current_account.key member.put() flask.session['current_tenant'] = member.tenant.urlsafe() flasher.success(_('Invite accepted')) return flask.redirect(flask.url_for('tenants.tenant_overview')) else: form = ChoosePasswordForm(flask.request.form) if form.validate_on_submit(): account, auth = UserAccount.from_email(member.invite_email, email_is_verified=True) auth.email_is_verified = True auth.email_verified_date = datetime.datetime.now() account.set_password(form.password.data) member.user = account.key put_multi([account, auth, member]) login_user(account) flasher.success(_('Invite accepted')) return flask.redirect(flask.url_for('tenants.tenant_overview')) flask.session['current_tenant'] = member.tenant.urlsafe() return flask.render_template('invite-accept-choose-password.html', form=form) else: return flask.render_template('no-invite-token.html'), 404
def test_put_hook(self): """Test put hooks.""" entity = TestModel3(prop_0=0) entity.put() self.assertEqual(1, entity.prop_1) self.assertEqual(entity.key.id(), entity.prop_2) self.assertDictEqual({ 'prop_0': 0, 'prop_1': 1, 'prop_2': None, }, entity.key.get().to_dict()) entities = [ TestModel3(id=entity.key.id(), prop_0=0), TestModel3(prop_0=1) ] ndb.put_multi(entities) self.assertEqual(1, entities[0].prop_1) self.assertEqual(entities[0].key.id(), entities[0].prop_2) self.assertEqual(2, entities[1].prop_1) self.assertEqual(entities[1].key.id(), entities[1].prop_2) self.assertItemsEqual([ { 'prop_0': 0, 'prop_1': 1, 'prop_2': None, }, { 'prop_0': 1, 'prop_1': 2, 'prop_2': None, }, ], [e.to_dict() for e in TestModel3.query()])
def _pseudonymize_models(activity_related_models, pseudonymized_user_id): """Pseudonymize user ID fields in the models. This function is run in a transaction, with the maximum number of activity_related_models being MAX_NUMBER_OF_OPS_IN_TRANSACTION. Args: activity_related_models: list(BaseModel). Models whose user IDs should be pseudonymized. pseudonymized_user_id: str. New pseudonymized user ID to be used for the models. """ metadata_models = [ model for model in activity_related_models if isinstance(model, snapshot_model_class)] for metadata_model in metadata_models: metadata_model.committer_id = pseudonymized_user_id commit_log_models = [ model for model in activity_related_models if isinstance(model, commit_log_model_class)] for commit_log_model in commit_log_models: commit_log_model.user_id = pseudonymized_user_id ndb.put_multi(metadata_models + commit_log_models)
def attach_tags(self, package_name, instance_id, tags, caller, now=None): """Adds a bunch of tags to an existing package instance. Idempotent. Skips existing tags. Package instance must exist and must have all processors successfully finished. Args: package_name: name of the package, e.g. 'infra/tools/cipd'. instance_id: identifier of the package instance (SHA1 of package file). tags: list of strings with tags to attach. caller: auth.Identity that issued the request. now: datetime when the request was made (or None for current time). Returns: {tag: corresponding InstanceTag (just created or existing one)}. """ assert tags and all(is_valid_instance_tag(tag) for tag in tags), tags self._assert_instance_is_ready(package_name, instance_id) # Grab info about existing tags, register new ones. now = now or utils.utcnow() existing = ndb.get_multi( instance_tag_key(package_name, instance_id, tag) for tag in tags) to_create = [ InstanceTag(key=instance_tag_key(package_name, instance_id, tag), tag=tag, registered_by=caller, registered_ts=now) for tag, ent in zip(tags, existing) if not ent ] ndb.put_multi(to_create) attached = {} attached.update({e.tag: e for e in existing if e}) attached.update({e.tag: e for e in to_create}) return attached
def testGetGraphJson_WithSelectedTrace(self): self._AddTestColumns(start_rev=15000, end_rev=15050) rows = graph_data.Row.query( graph_data.Row.parent_test == utils.OldStyleTestKey( 'ChromiumGPU/win7/dromaeo/jslib')).fetch() for row in rows: row.error = 1 + ((row.revision - 15000) * 0.25) ndb.put_multi(rows) flot_json_str = graph_json.GetGraphJson( { 'ChromiumGPU/win7/dromaeo/jslib': ['jslib'], }, rev=15000, num_points=8, is_selected=True) flot = json.loads(flot_json_str) self.assertEqual(1, len(flot['data'])) self.assertEqual(5, len(flot['data']['0']['data'])) self.assertEqual(1, len(flot['annotations']['series'])) self.assertEqual(5, len(flot['annotations'].get('0').keys())) self.assertEqual(5, len(flot['error_bars']['0'][0]['data'])) self.assertEqual(5, len(flot['error_bars']['0'][1]['data']))
def test_run_result_timeout(self): request = task_request.make_request(_gen_request_data()) result_summary = task_result.new_result_summary(request) result_summary.modified_ts = utils.utcnow() ndb.transaction(result_summary.put) run_result = task_result.new_run_result(request, 1, 'localhost', 'abc') run_result.state = task_result.State.TIMED_OUT run_result.completed_ts = utils.utcnow() run_result.modified_ts = utils.utcnow() result_summary.set_from_run_result(run_result, request) ndb.transaction(lambda: ndb.put_multi((run_result, result_summary))) run_result = run_result.key.get() result_summary = result_summary.key.get() self.assertEqual(True, run_result.failure) self.assertEqual(True, result_summary.failure)
def _CreateRows(self): test_path = 'Chromium/win7/suite/metric' test_key = utils.TestKey(test_path) stat_names_to_test_keys = { 'avg': utils.TestKey('Chromium/win7/suite/metric_avg'), 'std': utils.TestKey('Chromium/win7/suite/metric_std'), 'count': utils.TestKey('Chromium/win7/suite/metric_count'), 'max': utils.TestKey('Chromium/win7/suite/metric_max'), 'min': utils.TestKey('Chromium/win7/suite/metric_min'), 'sum': utils.TestKey('Chromium/win7/suite/metric_sum') } histograms = add_histograms_queue_test.TEST_HISTOGRAM histograms['diagnostics'][ reserved_infos.CHROMIUM_COMMIT_POSITIONS.name]['values'] = [99] ndb.put_multi( add_histograms_queue.CreateRowEntities(histograms, test_key, stat_names_to_test_keys, 99)) histograms['diagnostics'][ reserved_infos.CHROMIUM_COMMIT_POSITIONS.name]['values'] = [200] ndb.put_multi( add_histograms_queue.CreateRowEntities( add_histograms_queue_test.TEST_HISTOGRAM, test_key, stat_names_to_test_keys, 200))
def delete_messages_for(cls, username, keys): """Deletes messages for username Args: username: username of the user who to delete messages for keys: list of Message keys to delete Returns: A list of keys for the deleted messages """ msgs = ndb.get_multi(keys) for m in msgs: m.deleted_for = username return ndb.put_multi(msgs)
def setUp(self): super(TestOutput, self).setUp() request = _gen_request() result_summary = task_result.new_result_summary(request) result_summary.modified_ts = utils.utcnow() ndb.transaction(result_summary.put) to_run = task_to_run.new_task_to_run(request, 1, 0) self.run_result = task_result.new_run_result(request, to_run, 'localhost', 'abc', {}) self.run_result.started_ts = result_summary.modified_ts self.run_result.modified_ts = utils.utcnow() ndb.transaction(lambda: result_summary.set_from_run_result( self.run_result, request)) ndb.transaction(lambda: ndb.put_multi( (result_summary, self.run_result))) self.run_result = self.run_result.key.get()
def get(self): original_url = self.request.headers['Referer'] dellsts = self.request.get_all("status") if (len(dellsts) > 0): streams = Stream.query( Stream.name.IN(dellsts), Stream.author == users.get_current_user()).fetch() for stream in streams: pictures = db.GqlQuery( "SELECT * FROM Picture " + "WHERE ANCESTOR IS :1", db.Key.from_path('Stream', stream.name)) db.delete(pictures) ndb.delete_multi(ndb.put_multi(streams)) self.redirect(original_url)
def store_context(context): """Persist a furious.context.Context object to the datastore by loading it into a FuriousContext ndb.Model. """ logging.debug("Attempting to store Context %s.", context.id) entity = FuriousContext.from_context(context) # TODO: Handle exceptions and retries here. marker = FuriousCompletionMarker(id=context.id) key, _ = ndb.put_multi((entity, marker)) logging.debug("Stored Context with key: %s.", key) return key
def _DetectTreeClosureForTree(tree_name, all_statuses): """Detects tree closures for the given tree, and return the number.""" tree_closures = [] index = 0 previous_closure_complete = False while index < len(all_statuses): # Skip all leading open statuses to find the next closure. if not all_statuses[index].closed: index += 1 continue close_index = index index += 1 first_open_index = None # Skip all non-open status (close, throttle) to find the next open status. while index < len(all_statuses): if not all_statuses[index].closed: first_open_index = index break index += 1 latest_open_index = first_open_index # Skip to the most recent open status for this tree closure. while index < len(all_statuses): if all_statuses[index].closed: break latest_open_index = index index += 1 # Skip if no matching open status is found. if first_open_index is not None: # The identified closure might not be complete with all the open statuses. tree_closures.append( _CreateTreeClosure( tree_name, all_statuses[close_index:latest_open_index + 1], all_statuses[first_open_index])) else: # The previous tree closure was complete with all the open statuses, # because a new tree closure started and was incomplete. previous_closure_complete = True if not previous_closure_complete: tree_closures = tree_closures[:-1] # Save all the closures to datastore. return len(ndb.put_multi(tree_closures))