def MakeDirty(request):
"""For testing purposes, make some tests dirty."""
query = ResultParent.all()
result_times = []
for result_parent in query.fetch(10):
for result_time in ResultTime.all().ancestor(result_parent).fetch(1000):
result_time.dirty = True
result_times.append(result_time)
db.put(result_times)
return http.HttpResponse('Made %s result_times dirty' % len(result_times))
def MakeDirty(request):
"""For testing purposes, make some tests dirty."""
query = ResultParent.all()
result_times = []
for result_parent in query.fetch(10):
for result_time in ResultTime.all().ancestor(result_parent).fetch(
1000):
result_time.dirty = True
result_times.append(result_time)
db.put(result_times)
return http.HttpResponse('Made %s result_times dirty' % len(result_times))
def UpdateOldDirty():
"""Update dirty queries from the past."""
num_scheduled = 0
seen_result_parent_keys = set()
dirty_query = ResultTime.all(keys_only=True).filter('dirty =', True)
for i, result_time_key in enumerate(dirty_query.fetch(500)):
result_parent_key = result_time_key.parent()
if result_parent_key not in seen_result_parent_keys:
seen_result_parent_keys.add(result_parent_key)
result_parent = ResultParent.get(result_parent_key)
category = result_parent.category
age = datetime.datetime.now() - result_parent.created
if age.days > 0 or age.seconds > OLD_SECONDS:
logging.info(
'Schedule old dirty:%d:%d: %s, age=%s, result_parent=%s, result_time=%s',
i, num_scheduled, category, age, result_parent_key, result_time_key)
if ResultParent.ScheduleUpdateDirty(
result_time_key, category, count=-1):
num_scheduled += 1
if num_scheduled == 10:
break
def UpdateOldDirty():
"""Update dirty queries from the past."""
num_scheduled = 0
seen_result_parent_keys = set()
dirty_query = ResultTime.all(keys_only=True).filter('dirty =', True)
for i, result_time_key in enumerate(dirty_query.fetch(500)):
result_parent_key = result_time_key.parent()
if result_parent_key not in seen_result_parent_keys:
seen_result_parent_keys.add(result_parent_key)
result_parent = ResultParent.get(result_parent_key)
category = result_parent.category
age = datetime.datetime.now() - result_parent.created
if age.days > 0 or age.seconds > OLD_SECONDS:
logging.info(
'Schedule old dirty:%d:%d: %s, age=%s, result_parent=%s, result_time=%s',
i, num_scheduled, category, age, result_parent_key,
result_time_key)
if ResultParent.ScheduleUpdateDirty(result_time_key,
category,
count=-1):
num_scheduled += 1
if num_scheduled == 10:
break
def UpdateDirty(request):
"""Updates any dirty tests, adding its score to the appropriate ranker."""
logging.debug('UpdateDirty start.')
task_name_prefix = request.REQUEST.get('task_name_prefix', '')
result_time_key = request.REQUEST.get('result_time_key')
category = request.REQUEST.get('category')
count = int(request.REQUEST.get('count', 0))
if result_time_key:
result_time = ResultTime.get(result_time_key)
try:
ResultTime.UpdateStats(result_time)
except:
logging.info('UpdateStats: %s:%s' % (sys.exc_type, sys.exc_value))
result_parent_key = result_time.parent_key()
else:
result_parent_key = request.REQUEST.get('result_parent_key')
if result_parent_key:
result_parent_key = db.Key(result_parent_key)
else:
UpdateOldDirty()
return http.HttpResponse('Done scheduling old results.')
# Create a task for the next dirty ResultTime to update.
dirty_query = ResultTime.all(keys_only=True)
dirty_query.filter('dirty =', True)
dirty_query.ancestor(result_parent_key)
next_result_time_key = dirty_query.get()
if next_result_time_key:
logging.debug('Schedule next ResultTime: %s', next_result_time_key)
ResultParent.ScheduleUpdateDirty(next_result_time_key, category,
count + 1, task_name_prefix)
else:
logging.debug('Done with result_parent: %s', result_parent_key)
ScheduleCategoryUpdate(result_parent_key)
shardedcounter.increment(category)
return http.HttpResponse('Done.')
def UpdateDirty(request):
"""Updates any dirty tests, adding its score to the appropriate ranker."""
logging.debug('UpdateDirty start.')
task_name_prefix = request.REQUEST.get('task_name_prefix', '')
result_time_key = request.REQUEST.get('result_time_key')
category = request.REQUEST.get('category')
count = int(request.REQUEST.get('count', 0))
if result_time_key:
result_time = ResultTime.get(result_time_key)
try:
ResultTime.UpdateStats(result_time)
except:
logging.info('UpdateStats: %s:%s' % (sys.exc_type, sys.exc_value))
result_parent_key = result_time.parent_key()
else:
result_parent_key = request.REQUEST.get('result_parent_key')
if result_parent_key:
result_parent_key = db.Key(result_parent_key)
else:
UpdateOldDirty()
return http.HttpResponse('Done scheduling old results.')
# Create a task for the next dirty ResultTime to update.
dirty_query = ResultTime.all(keys_only=True)
dirty_query.filter('dirty =', True)
dirty_query.ancestor(result_parent_key)
next_result_time_key = dirty_query.get()
if next_result_time_key:
logging.debug('Schedule next ResultTime: %s', next_result_time_key)
ResultParent.ScheduleUpdateDirty(
next_result_time_key, category, count+1, task_name_prefix)
else:
logging.debug('Done with result_parent: %s', result_parent_key)
ScheduleCategoryUpdate(result_parent_key)
shardedcounter.increment(category)
return http.HttpResponse('Done.')