def rebuild_love_count(): utc_dt = datetime.datetime.utcnow() - datetime.timedelta( days=7) # rebuild last week and this week week_start, _ = utc_week_limits(utc_dt) set_toggle_state(LOVE_SENDING_ENABLED, False) logging.info('Deleting LoveCount table...') ndb.delete_multi( LoveCount.query(LoveCount.week_start >= week_start).fetch( keys_only=True)) employee_dict = {employee.key: employee for employee in Employee.query()} logging.info('Rebuilding LoveCount table...') cursor = None count = 0 while True: loves, cursor, has_more = Love.query( Love.timestamp >= week_start).fetch_page(500, start_cursor=cursor) for l in loves: LoveCount.update(l, employee_dict=employee_dict) count += len(loves) logging.info('Processed {} loves'.format(count)) if not has_more: break logging.info('Done.') set_toggle_state(LOVE_SENDING_ENABLED, True)
def batch_fuzzer_jobs(): """Batch FuzzerJobs for reduced Datastore read ops by bots.""" platforms = [ item.platform for item in data_types.FuzzerJob.query( projection=[data_types.FuzzerJob.platform], distinct=True) ] for platform in platforms: fuzzer_jobs = list( data_types.FuzzerJob.query( data_types.FuzzerJob.platform == platform)) fuzzer_jobs.sort(key=lambda item: item.job) batches_to_remove = set(b.key for b in data_types.FuzzerJobs.query( data_types.FuzzerJobs.platform == platform)) batch_count = 0 for i in range(0, len(fuzzer_jobs), FUZZER_JOB_BATCH_SIZE): key_id = platform + '-' + str(batch_count) end = min(i + FUZZER_JOB_BATCH_SIZE, len(fuzzer_jobs)) batched = data_types.FuzzerJobs(id=key_id, platform=platform) batched.platform = platform batched.fuzzer_jobs = fuzzer_jobs[i:end] batched.put() batch_count += 1 batches_to_remove.discard(batched.key) # Remove additional batches if number reduced. if batches_to_remove: ndb.delete_multi(batches_to_remove)
def update_affected_commits(bug_id, result, project, ecosystem, public): """Update affected commits.""" to_put = [] to_delete = [] for commit in result.commits: affected_commit = osv.AffectedCommit( id=bug_id + '-' + commit, bug_id=bug_id, commit=commit, confidence=result.confidence, project=project, ecosystem=ecosystem, public=public) to_put.append(affected_commit) # Delete any affected commits that no longer apply. This can happen in cases # where a FixResult comes in later and we had previously marked a commit prior # to the fix commit as being affected by a vulnerability. for existing in osv.AffectedCommit.query(osv.AffectedCommit.bug_id == bug_id): if existing.commit not in result.commits: to_delete.append(existing.key) ndb.put_multi(to_put) ndb.delete_multi(to_delete)
def remove_unverified_users(cls): with client.context(): users_keys = cls.query( cls.verification_code != "", cls.verification_code_expiration < datetime.datetime.now()).fetch(keys_only=True) ndb.delete_multi(keys=users_keys) return True
def post(self): type = self.request.get('type', '') if type == 'stats': pass elif type == 'cleanup': last_year = datetime.datetime.now() - datetime.timedelta(days=365) last_quarter = datetime.datetime.now() - datetime.timedelta( days=92) last_month = datetime.datetime.now() - datetime.timedelta(days=31) # Old news old_news = News.query(News.date < last_quarter).order( News.date).fetch(500, keys_only=True) # logging.info('Cleaning up old news %s' % News.query().order(News.date).count(100,keys_only=True)) ndb.delete_multi(old_news) elif type == 'tag_cloud': channel_urls = [] tags = {} extras = Extra.query(Extra.tag != None) for extra in extras: if extra.channelurl not in channel_urls: channel_urls.append(extra.channelurl) tag = extra.tag if tag in tags: tags[tag] += 1 else: tags[tag] = 1 tags_sorted = sorted(tags.iteritems(), key=operator.itemgetter(1), reverse=True) memcache.set("tag_cloud", tags_sorted) logging.debug('Tags: %s' % (tags_sorted)) elif type == 'fix': test_channel = '#kanava' channel = Channel.query(Channel.name == test_channel).get() channelurls = ChannelUrl.query( ChannelUrl.channel == channel.key).fetch(50) for channelurl in channelurls: url = channelurl.url.get() logging.debug('Channel: %s, channelurl: %s (id %s)' % (test_channel, url, channelurl)) posts = Post.query(Post.channelurl == channelurl.key) for post in posts: logging.debug(' * posted by %s' % (post.user)) post.key.delete() rates = Rate.query(Rate.channelurl == channelurl.key) for rate in rates: logging.debug(' * rate %s' % (rate)) rate.key.delete() extras = Extra.query(Extra.channelurl == channelurl.key) for extra in extras: logging.debug(' * extra %s, by %s' % (extra, extra.user)) extra.key.delete() channelurl.key.delete()
def purge_all(): keys = [r.key for r in Room.query()] try: ndb.delete_multi(keys) resp = 'All rooms deleted' except Exception as ex: resp = f'Error purging rooms: {ex}' return resp
def permanently_batch_delete(cls): # Permanently delete users that were marked as deleted=True more than 30 days ago with client.context(): users_keys = cls.query( cls.deleted == True, cls.deleted_date < (datetime.datetime.now() - datetime.timedelta(days=30))).fetch( keys_only=True) ndb.delete_multi(keys=users_keys) return True
def clear_expireds(cls) -> int: """Remove all member candidiates from the datastore that have expired. Returns the count of records removed. """ now = datetime.datetime.now() with cls._ndb_client.context(): expireds = MemberCandidate.query( MemberCandidate.expire <= now).fetch() if not expireds: return 0 ndb.delete_multi([expired.key for expired in expireds]) return len(expireds)
def mark_bug_invalid(message): """Mark a bug as invalid.""" source_id = get_source_id(message) bug = osv.Bug.query(osv.Bug.source_id == source_id).get() if not bug: logging.error('Bug with source id %s does not exist.', source_id) return bug.status = osv.BugStatus.INVALID bug.put() affected_commits = osv.AffectedCommit.query( osv.AffectedCommit.bug_id == bug.key.id()) ndb.delete_multi([commit.key for commit in affected_commits])
def love_links_cleanup(): """ Deletes love links that are more than a month (30 days) old. """ earliest = datetime.datetime.now() - datetime.timedelta(days=30) love_links_keys = LoveLink.query(LoveLink.timestamp <= earliest).fetch( keys_only=True) logging.info('Preparing to delete love links older than {}.'.format( str(earliest))) ndb.delete_multi(love_links_keys) logging.info('Love links older than {} were deleted.'.format( str(earliest))) return
def trim(): '(push) task queue handler to delete oldest visits' oldest = float(request.get_json().get('oldest')) with ds_client.context(): keys = Visit.query( Visit.timestamp < datetime.fromtimestamp(oldest)).fetch( keys_only=True) nkeys = len(keys) if nkeys: logging.info('Deleting %d entities: %s' % (nkeys, ', '.join(str(k.id()) for k in keys))) ndb.delete_multi(keys) else: logging.info('No entities older than: %s' % time.ctime(oldest)) return '' # need to return SOME string w/200
def sync_projects(projects): """Sync projects with cloud datastore.""" project_query = Project.query() projects_to_remove = [ project.key for project in project_query if project.name not in projects ] ndb.delete_multi(projects_to_remove) existing_projects = {project.name for project in project_query} new_projects = [ Project(name=project) for project in projects if project not in existing_projects ] ndb.put_multi(new_projects)
def delete_multi(keys: Sequence[Key]) -> List[None]: """Deletes models corresponding to a sequence of keys. Args: keys: list(str). A list of keys. Returns: list(None). A list of Nones, one per deleted model. """ return ndb.delete_multi(keys)
def delete_multi_transactional(keys: List[Key]) -> List[None]: """Deletes models corresponding to a sequence of keys and runs it through a transaction. Either all models are deleted, or none of them in the case when the transaction fails. Args: keys: list(str). A list of keys. Returns: list(None). A list of Nones, one per deleted model. """ return ndb.delete_multi(keys)
def test_multi_with_lots_of_keys(dispose_of): """Regression test for issue #318. https://github.com/googleapis/python-ndb/issues/318 """ N = 1001 class SomeKind(ndb.Model): foo = ndb.IntegerProperty() foos = list(range(N)) entities = [SomeKind(foo=foo) for foo in foos] keys = ndb.put_multi(entities) dispose_of(*(key._key for key in keys)) assert len(keys) == N entities = ndb.get_multi(keys) assert [entity.foo for entity in entities] == foos ndb.delete_multi(keys) entities = ndb.get_multi(keys) assert entities == [None] * N
def process_export(old_export_path, new_export_path): for table, cls in get_tables(): logging.info('Processing ' + table) table_suffix = '/WCA_export_' + table + '.tsv' old_rows = read_table(old_export_path + table_suffix, cls, False) new_rows = read_table(new_export_path + table_suffix, cls, True) logging.info('Old: %d' % len(old_rows)) logging.info('New: %d' % len(new_rows)) write_table(new_export_path + table_suffix, new_rows, cls) modifier = get_modifier(table) objects_to_put = [] keys_to_delete = [] for key in new_rows: row = new_rows[key] if key in old_rows and old_rows[key] == row: continue else: obj = cls(id=key) obj.ParseFromDict(row) if modifier: modifier(obj) objects_to_put += [obj] for key, row in old_rows.items(): if key in new_rows: continue else: keys_to_delete += [ndb.Key(cls, key)] logging.info('Putting %d objects' % len(objects_to_put)) while objects_to_put: batch_size = 500 subslice = objects_to_put[:batch_size] objects_to_put = objects_to_put[batch_size:] ndb.put_multi(subslice) logging.info('Deleting %d objects' % len(keys_to_delete)) ndb.delete_multi(keys_to_delete)
def _throttled_delete(to_delete): """Throttled ndb delete.""" for batch, is_last in _batcher(to_delete, _DATASTORE_BATCH_SIZE): ndb.delete_multi(batch) if not is_last: time.sleep(_DATASTORE_BATCH_SLEEP)
def operate_on_multiple_keys_at_once(list_of_entities): list_of_keys = ndb.put_multi(list_of_entities) list_of_entities = ndb.get_multi(list_of_keys) ndb.delete_multi(list_of_keys)
from google.cloud import ndb from app.models.wca.export import get_latest_export client = ndb.Client() # Delete classes we don't use anymore. with client.context(): for clsname in ['AppSettings', 'Document', 'Schedule', 'ScheduleCompetition', 'SchedulePerson', 'ScheduleRound', 'ScheduleStaff', 'ScheduleStage', 'ScheduleTimeBlock', 'WcaExport']: class MyModel(ndb.Model): pass MyModel.__name__ = clsname ndb.delete_multi(MyModel.query().fetch(keys_only=True))
def UpdateChampions(): champions_to_write = [] champions_to_delete = [] final_round_keys = set( r.key for r in RoundType.query(RoundType.final == True).iter()) all_event_keys = set(e.key for e in Event.query().iter()) championships_already_computed = set() for champion in Champion.query().iter(): championships_already_computed.add(champion.championship.id()) for championship in Championship.query().iter(): if not championship.national_championship and os.environ.get( 'ENV') == 'DEV': # Don't try to compute regional champions on dev, since we don't have # location data. continue competition = championship.competition.get() # Only recompute champions from the last 2 weeks, in case there are result updates. if (championship.key.id() in championships_already_computed and datetime.date.today() - competition.end_date > datetime.timedelta(days=14)): continue if competition.end_date > datetime.date.today(): continue competition_id = championship.competition.id() logging.info('Computing champions for %s' % competition_id) results = (Result.query( Result.competition == championship.competition).order( Result.pos).fetch()) if not results: logging.info( 'Results are not uploaded yet. Not computing champions yet.') continue eligible_competitors = ComputeEligibleCompetitors( championship, competition, results) champions = collections.defaultdict(list) events_held_with_successes = set() for result in results: if result.best < 0: continue if result.round_type not in final_round_keys: continue this_event = result.event # For multi blind, we only recognize pre-2009 champions in 333mbo, since # that was the multi-blind event held those years. For clarity in the # champions listings, we list those champions as the 333mbf champions for # those years. if championship.competition.get().year < 2009: if result.event.id() == '333mbo': this_event = ndb.Key(Event, '333mbf') elif result.event.id() == '333mbf': continue events_held_with_successes.add(this_event) if this_event in champions and champions[this_event][ 0].pos < result.pos: continue if result.person.id() not in eligible_competitors: continue champions[this_event].append(result) if result.pos > 1 and len(champions) >= len( events_held_with_successes): break for event_key in all_event_keys: champion_id = Champion.Id(championship.key.id(), event_key.id()) if event_key in champions: champion = Champion.get_by_id(champion_id) or Champion( id=champion_id) champion.championship = championship.key champion.event = event_key champion.champions = [c.key for c in champions[event_key]] champions_to_write.append(champion) else: champions_to_delete.append(ndb.Key(Champion, champion_id)) ndb.put_multi(champions_to_write) ndb.delete_multi(champions_to_delete)
def delete_them(entities): ndb.delete_multi([entity.key for entity in entities])
def combine_employees(old_username, new_username): set_toggle_state(LOVE_SENDING_ENABLED, False) old_employee_key = Employee.query(Employee.username == old_username).get(keys_only=True) new_employee_key = Employee.query(Employee.username == new_username).get(keys_only=True) if not old_employee_key: raise NoSuchEmployee(old_username) elif not new_employee_key: raise NoSuchEmployee(new_username) # First, we need to update the actual instances of Love sent to/from the old employee logging.info('Reassigning {}\'s love to {}...'.format(old_username, new_username)) love_to_save = [] # Reassign all love sent FROM old_username for sent_love in Love.query(Love.sender_key == old_employee_key).iter(): sent_love.sender_key = new_employee_key love_to_save.append(sent_love) # Reassign all love sent TO old_username for received_love in Love.query(Love.recipient_key == old_employee_key).iter(): received_love.recipient_key = new_employee_key love_to_save.append(received_love) ndb.put_multi(love_to_save) logging.info('Done reassigning love.') # Second, we need to update the LoveCount table logging.info('Updating LoveCount table...') love_counts_to_delete, love_counts_to_save = [], [] for old_love_count in LoveCount.query(ancestor=old_employee_key).iter(): # Try to find a corresponding row for the new employee new_love_count = LoveCount.query( ancestor=new_employee_key).filter(LoveCount.week_start == old_love_count.week_start).get() if new_love_count is None: # If there's no corresponding row for the new user, create one new_love_count = LoveCount( parent=new_employee_key, received_count=old_love_count.received_count, sent_count=old_love_count.sent_count, week_start=old_love_count.week_start ) else: # Otherwise, combine the two rows new_love_count.received_count += old_love_count.received_count new_love_count.sent_count += old_love_count.sent_count # You `delete` keys but you `put` entities... Google's APIs are weird love_counts_to_delete.append(old_love_count.key) love_counts_to_save.append(new_love_count) ndb.delete_multi(love_counts_to_delete) ndb.put_multi(love_counts_to_save) logging.info('Done updating LoveCount table.') # Now we can delete the old employee logging.info('Deleting employee {}...'.format(old_username)) old_employee_key.delete() logging.info('Done deleting employee.') set_toggle_state(LOVE_SENDING_ENABLED, True)
def delete_multi(keys): """Delete multiple entities, working around a limitation in the NDB library with the maximum number of keys allowed.""" for chunk in _gen_chunks(keys, _MODIFY_BATCH_SIZE): ndb.delete_multi(chunk)