def rebuild_love_count():
utc_dt = datetime.datetime.utcnow() - datetime.timedelta(
days=7) # rebuild last week and this week
week_start, _ = utc_week_limits(utc_dt)
set_toggle_state(LOVE_SENDING_ENABLED, False)
logging.info('Deleting LoveCount table...')
ndb.delete_multi(
LoveCount.query(LoveCount.week_start >= week_start).fetch(
keys_only=True))
employee_dict = {employee.key: employee for employee in Employee.query()}
logging.info('Rebuilding LoveCount table...')
cursor = None
count = 0
while True:
loves, cursor, has_more = Love.query(
Love.timestamp >= week_start).fetch_page(500, start_cursor=cursor)
for l in loves:
LoveCount.update(l, employee_dict=employee_dict)
count += len(loves)
logging.info('Processed {} loves'.format(count))
if not has_more:
break
logging.info('Done.')
set_toggle_state(LOVE_SENDING_ENABLED, True)
def batch_fuzzer_jobs():
"""Batch FuzzerJobs for reduced Datastore read ops by bots."""
platforms = [
item.platform for item in data_types.FuzzerJob.query(
projection=[data_types.FuzzerJob.platform], distinct=True)
]
for platform in platforms:
fuzzer_jobs = list(
data_types.FuzzerJob.query(
data_types.FuzzerJob.platform == platform))
fuzzer_jobs.sort(key=lambda item: item.job)
batches_to_remove = set(b.key for b in data_types.FuzzerJobs.query(
data_types.FuzzerJobs.platform == platform))
batch_count = 0
for i in range(0, len(fuzzer_jobs), FUZZER_JOB_BATCH_SIZE):
key_id = platform + '-' + str(batch_count)
end = min(i + FUZZER_JOB_BATCH_SIZE, len(fuzzer_jobs))
batched = data_types.FuzzerJobs(id=key_id, platform=platform)
batched.platform = platform
batched.fuzzer_jobs = fuzzer_jobs[i:end]
batched.put()
batch_count += 1
batches_to_remove.discard(batched.key)
# Remove additional batches if number reduced.
if batches_to_remove:
ndb.delete_multi(batches_to_remove)
def update_affected_commits(bug_id, result, project, ecosystem, public):
"""Update affected commits."""
to_put = []
to_delete = []
for commit in result.commits:
affected_commit = osv.AffectedCommit(
id=bug_id + '-' + commit,
bug_id=bug_id,
commit=commit,
confidence=result.confidence,
project=project,
ecosystem=ecosystem,
public=public)
to_put.append(affected_commit)
# Delete any affected commits that no longer apply. This can happen in cases
# where a FixResult comes in later and we had previously marked a commit prior
# to the fix commit as being affected by a vulnerability.
for existing in osv.AffectedCommit.query(osv.AffectedCommit.bug_id == bug_id):
if existing.commit not in result.commits:
to_delete.append(existing.key)
ndb.put_multi(to_put)
ndb.delete_multi(to_delete)
def remove_unverified_users(cls):
with client.context():
users_keys = cls.query(
cls.verification_code != "", cls.verification_code_expiration <
datetime.datetime.now()).fetch(keys_only=True)
ndb.delete_multi(keys=users_keys)
return True
def post(self):
type = self.request.get('type', '')
if type == 'stats':
pass
elif type == 'cleanup':
last_year = datetime.datetime.now() - datetime.timedelta(days=365)
last_quarter = datetime.datetime.now() - datetime.timedelta(
days=92)
last_month = datetime.datetime.now() - datetime.timedelta(days=31)
# Old news
old_news = News.query(News.date < last_quarter).order(
News.date).fetch(500, keys_only=True)
# logging.info('Cleaning up old news %s' % News.query().order(News.date).count(100,keys_only=True))
ndb.delete_multi(old_news)
elif type == 'tag_cloud':
channel_urls = []
tags = {}
extras = Extra.query(Extra.tag != None)
for extra in extras:
if extra.channelurl not in channel_urls:
channel_urls.append(extra.channelurl)
tag = extra.tag
if tag in tags:
tags[tag] += 1
else:
tags[tag] = 1
tags_sorted = sorted(tags.iteritems(),
key=operator.itemgetter(1),
reverse=True)
memcache.set("tag_cloud", tags_sorted)
logging.debug('Tags: %s' % (tags_sorted))
elif type == 'fix':
test_channel = '#kanava'
channel = Channel.query(Channel.name == test_channel).get()
channelurls = ChannelUrl.query(
ChannelUrl.channel == channel.key).fetch(50)
for channelurl in channelurls:
url = channelurl.url.get()
logging.debug('Channel: %s, channelurl: %s (id %s)' %
(test_channel, url, channelurl))
posts = Post.query(Post.channelurl == channelurl.key)
for post in posts:
logging.debug(' * posted by %s' % (post.user))
post.key.delete()
rates = Rate.query(Rate.channelurl == channelurl.key)
for rate in rates:
logging.debug(' * rate %s' % (rate))
rate.key.delete()
extras = Extra.query(Extra.channelurl == channelurl.key)
for extra in extras:
logging.debug(' * extra %s, by %s' % (extra, extra.user))
extra.key.delete()
channelurl.key.delete()
def purge_all():
keys = [r.key for r in Room.query()]
try:
ndb.delete_multi(keys)
resp = 'All rooms deleted'
except Exception as ex:
resp = f'Error purging rooms: {ex}'
return resp
def permanently_batch_delete(cls):
# Permanently delete users that were marked as deleted=True more than 30 days ago
with client.context():
users_keys = cls.query(
cls.deleted == True, cls.deleted_date <
(datetime.datetime.now() - datetime.timedelta(days=30))).fetch(
keys_only=True)
ndb.delete_multi(keys=users_keys)
return True
def clear_expireds(cls) -> int:
"""Remove all member candidiates from the datastore that have expired.
Returns the count of records removed.
"""
now = datetime.datetime.now()
with cls._ndb_client.context():
expireds = MemberCandidate.query(
MemberCandidate.expire <= now).fetch()
if not expireds:
return 0
ndb.delete_multi([expired.key for expired in expireds])
return len(expireds)
def mark_bug_invalid(message):
"""Mark a bug as invalid."""
source_id = get_source_id(message)
bug = osv.Bug.query(osv.Bug.source_id == source_id).get()
if not bug:
logging.error('Bug with source id %s does not exist.', source_id)
return
bug.status = osv.BugStatus.INVALID
bug.put()
affected_commits = osv.AffectedCommit.query(
osv.AffectedCommit.bug_id == bug.key.id())
ndb.delete_multi([commit.key for commit in affected_commits])
def love_links_cleanup():
"""
Deletes love links that are more than a month (30 days) old.
"""
earliest = datetime.datetime.now() - datetime.timedelta(days=30)
love_links_keys = LoveLink.query(LoveLink.timestamp <= earliest).fetch(
keys_only=True)
logging.info('Preparing to delete love links older than {}.'.format(
str(earliest)))
ndb.delete_multi(love_links_keys)
logging.info('Love links older than {} were deleted.'.format(
str(earliest)))
return
def trim():
'(push) task queue handler to delete oldest visits'
oldest = float(request.get_json().get('oldest'))
with ds_client.context():
keys = Visit.query(
Visit.timestamp < datetime.fromtimestamp(oldest)).fetch(
keys_only=True)
nkeys = len(keys)
if nkeys:
logging.info('Deleting %d entities: %s' %
(nkeys, ', '.join(str(k.id()) for k in keys)))
ndb.delete_multi(keys)
else:
logging.info('No entities older than: %s' % time.ctime(oldest))
return '' # need to return SOME string w/200
def sync_projects(projects):
"""Sync projects with cloud datastore."""
project_query = Project.query()
projects_to_remove = [
project.key for project in project_query
if project.name not in projects
]
ndb.delete_multi(projects_to_remove)
existing_projects = {project.name for project in project_query}
new_projects = [
Project(name=project) for project in projects
if project not in existing_projects
]
ndb.put_multi(new_projects)
def delete_multi(keys: Sequence[Key]) -> List[None]:
"""Deletes models corresponding to a sequence of keys.
Args:
keys: list(str). A list of keys.
Returns:
list(None). A list of Nones, one per deleted model.
"""
return ndb.delete_multi(keys)
def delete_multi_transactional(keys: List[Key]) -> List[None]:
"""Deletes models corresponding to a sequence of keys and runs it through
a transaction. Either all models are deleted, or none of them in the case
when the transaction fails.
Args:
keys: list(str). A list of keys.
Returns:
list(None). A list of Nones, one per deleted model.
"""
return ndb.delete_multi(keys)
def test_multi_with_lots_of_keys(dispose_of):
"""Regression test for issue #318.
https://github.com/googleapis/python-ndb/issues/318
"""
N = 1001
class SomeKind(ndb.Model):
foo = ndb.IntegerProperty()
foos = list(range(N))
entities = [SomeKind(foo=foo) for foo in foos]
keys = ndb.put_multi(entities)
dispose_of(*(key._key for key in keys))
assert len(keys) == N
entities = ndb.get_multi(keys)
assert [entity.foo for entity in entities] == foos
ndb.delete_multi(keys)
entities = ndb.get_multi(keys)
assert entities == [None] * N
def process_export(old_export_path, new_export_path):
for table, cls in get_tables():
logging.info('Processing ' + table)
table_suffix = '/WCA_export_' + table + '.tsv'
old_rows = read_table(old_export_path + table_suffix, cls, False)
new_rows = read_table(new_export_path + table_suffix, cls, True)
logging.info('Old: %d' % len(old_rows))
logging.info('New: %d' % len(new_rows))
write_table(new_export_path + table_suffix, new_rows, cls)
modifier = get_modifier(table)
objects_to_put = []
keys_to_delete = []
for key in new_rows:
row = new_rows[key]
if key in old_rows and old_rows[key] == row:
continue
else:
obj = cls(id=key)
obj.ParseFromDict(row)
if modifier:
modifier(obj)
objects_to_put += [obj]
for key, row in old_rows.items():
if key in new_rows:
continue
else:
keys_to_delete += [ndb.Key(cls, key)]
logging.info('Putting %d objects' % len(objects_to_put))
while objects_to_put:
batch_size = 500
subslice = objects_to_put[:batch_size]
objects_to_put = objects_to_put[batch_size:]
ndb.put_multi(subslice)
logging.info('Deleting %d objects' % len(keys_to_delete))
ndb.delete_multi(keys_to_delete)
def _throttled_delete(to_delete):
"""Throttled ndb delete."""
for batch, is_last in _batcher(to_delete, _DATASTORE_BATCH_SIZE):
ndb.delete_multi(batch)
if not is_last:
time.sleep(_DATASTORE_BATCH_SLEEP)
def operate_on_multiple_keys_at_once(list_of_entities):
list_of_keys = ndb.put_multi(list_of_entities)
list_of_entities = ndb.get_multi(list_of_keys)
ndb.delete_multi(list_of_keys)
from google.cloud import ndb
from app.models.wca.export import get_latest_export
client = ndb.Client()
# Delete classes we don't use anymore.
with client.context():
for clsname in ['AppSettings',
'Document',
'Schedule',
'ScheduleCompetition',
'SchedulePerson',
'ScheduleRound',
'ScheduleStaff',
'ScheduleStage',
'ScheduleTimeBlock',
'WcaExport']:
class MyModel(ndb.Model):
pass
MyModel.__name__ = clsname
ndb.delete_multi(MyModel.query().fetch(keys_only=True))
def UpdateChampions():
champions_to_write = []
champions_to_delete = []
final_round_keys = set(
r.key for r in RoundType.query(RoundType.final == True).iter())
all_event_keys = set(e.key for e in Event.query().iter())
championships_already_computed = set()
for champion in Champion.query().iter():
championships_already_computed.add(champion.championship.id())
for championship in Championship.query().iter():
if not championship.national_championship and os.environ.get(
'ENV') == 'DEV':
# Don't try to compute regional champions on dev, since we don't have
# location data.
continue
competition = championship.competition.get()
# Only recompute champions from the last 2 weeks, in case there are result updates.
if (championship.key.id() in championships_already_computed
and datetime.date.today() - competition.end_date >
datetime.timedelta(days=14)):
continue
if competition.end_date > datetime.date.today():
continue
competition_id = championship.competition.id()
logging.info('Computing champions for %s' % competition_id)
results = (Result.query(
Result.competition == championship.competition).order(
Result.pos).fetch())
if not results:
logging.info(
'Results are not uploaded yet. Not computing champions yet.')
continue
eligible_competitors = ComputeEligibleCompetitors(
championship, competition, results)
champions = collections.defaultdict(list)
events_held_with_successes = set()
for result in results:
if result.best < 0:
continue
if result.round_type not in final_round_keys:
continue
this_event = result.event
# For multi blind, we only recognize pre-2009 champions in 333mbo, since
# that was the multi-blind event held those years. For clarity in the
# champions listings, we list those champions as the 333mbf champions for
# those years.
if championship.competition.get().year < 2009:
if result.event.id() == '333mbo':
this_event = ndb.Key(Event, '333mbf')
elif result.event.id() == '333mbf':
continue
events_held_with_successes.add(this_event)
if this_event in champions and champions[this_event][
0].pos < result.pos:
continue
if result.person.id() not in eligible_competitors:
continue
champions[this_event].append(result)
if result.pos > 1 and len(champions) >= len(
events_held_with_successes):
break
for event_key in all_event_keys:
champion_id = Champion.Id(championship.key.id(), event_key.id())
if event_key in champions:
champion = Champion.get_by_id(champion_id) or Champion(
id=champion_id)
champion.championship = championship.key
champion.event = event_key
champion.champions = [c.key for c in champions[event_key]]
champions_to_write.append(champion)
else:
champions_to_delete.append(ndb.Key(Champion, champion_id))
ndb.put_multi(champions_to_write)
ndb.delete_multi(champions_to_delete)
def delete_them(entities):
ndb.delete_multi([entity.key for entity in entities])
def combine_employees(old_username, new_username):
set_toggle_state(LOVE_SENDING_ENABLED, False)
old_employee_key = Employee.query(Employee.username == old_username).get(keys_only=True)
new_employee_key = Employee.query(Employee.username == new_username).get(keys_only=True)
if not old_employee_key:
raise NoSuchEmployee(old_username)
elif not new_employee_key:
raise NoSuchEmployee(new_username)
# First, we need to update the actual instances of Love sent to/from the old employee
logging.info('Reassigning {}\'s love to {}...'.format(old_username, new_username))
love_to_save = []
# Reassign all love sent FROM old_username
for sent_love in Love.query(Love.sender_key == old_employee_key).iter():
sent_love.sender_key = new_employee_key
love_to_save.append(sent_love)
# Reassign all love sent TO old_username
for received_love in Love.query(Love.recipient_key == old_employee_key).iter():
received_love.recipient_key = new_employee_key
love_to_save.append(received_love)
ndb.put_multi(love_to_save)
logging.info('Done reassigning love.')
# Second, we need to update the LoveCount table
logging.info('Updating LoveCount table...')
love_counts_to_delete, love_counts_to_save = [], []
for old_love_count in LoveCount.query(ancestor=old_employee_key).iter():
# Try to find a corresponding row for the new employee
new_love_count = LoveCount.query(
ancestor=new_employee_key).filter(LoveCount.week_start == old_love_count.week_start).get()
if new_love_count is None:
# If there's no corresponding row for the new user, create one
new_love_count = LoveCount(
parent=new_employee_key,
received_count=old_love_count.received_count,
sent_count=old_love_count.sent_count,
week_start=old_love_count.week_start
)
else:
# Otherwise, combine the two rows
new_love_count.received_count += old_love_count.received_count
new_love_count.sent_count += old_love_count.sent_count
# You `delete` keys but you `put` entities... Google's APIs are weird
love_counts_to_delete.append(old_love_count.key)
love_counts_to_save.append(new_love_count)
ndb.delete_multi(love_counts_to_delete)
ndb.put_multi(love_counts_to_save)
logging.info('Done updating LoveCount table.')
# Now we can delete the old employee
logging.info('Deleting employee {}...'.format(old_username))
old_employee_key.delete()
logging.info('Done deleting employee.')
set_toggle_state(LOVE_SENDING_ENABLED, True)
def delete_multi(keys):
"""Delete multiple entities, working around a limitation in the NDB library
with the maximum number of keys allowed."""
for chunk in _gen_chunks(keys, _MODIFY_BATCH_SIZE):
ndb.delete_multi(chunk)