def _clear_ndb(): # pragma: no cover
for cq_stats in CQStats.query():
cq_stats.key.delete()
assert CQStats.query().count() == 0
for record in Record.query():
record.key.delete()
assert Record.query().count() == 0
def test_post_single_key_update(self):
self.mock_current_user(is_admin=True)
response = self.test_app.get('/post', params={
'key': 'single_key_update',
'fields': '{"project": "prj"}',
})
self.assertEquals('', response.body)
record = Record.get_by_id('single_key_update')
self.assertTrue(record != None)
self.assertEquals(['project=prj'], record.tags)
self.assertEquals({'project': 'prj'}, record.fields)
old_count = Record.query().count()
response = self.test_app.get('/post', params={
'key': 'single_key_update',
'tags': '1,2,3',
'fields': '{"update": "the", "same": "record", "project": "prj"}',
})
self.assertEquals('', response.body)
self.assertEquals(old_count, Record.query().count())
record = Record.get_by_id('single_key_update')
self.assertTrue(record != None)
self.assertEquals(set(['1', '2', '3', 'project=prj']), set(record.tags))
self.assertEquals({'update': 'the', 'same': 'record', 'project': 'prj'},
record.fields)
def test_post_multiple(self):
self.mock_current_user(is_admin=True)
old_count = Record.query().count()
response = self.test_app.post('/post', params=[
('p', json.dumps({
'key': 'multiple_0',
'tags': ['hello', 'world'],
'fields': {'hello': 'world', 'project': 'prj'},
})),
('p', json.dumps({
'key': 'multiple_1',
'fields': {'issue': 'autotagged', 'project': 'prj'},
})),
('p', json.dumps({
'key': 'multiple_2',
'tags': ['empty', 'fields'],
'fields': {'project': 'prj'},
})),
])
self.assertEquals('', response.body)
self.assertEquals(old_count + 3, Record.query().count())
record = Record.get_by_id('multiple_0')
self.assertEquals(set(['hello', 'world', 'project=prj']), set(record.tags))
self.assertEquals({'hello': 'world', 'project': 'prj'}, record.fields)
record = Record.get_by_id('multiple_1')
self.assertEquals(['issue=autotagged', 'project=prj'], record.tags)
self.assertEquals({'issue': 'autotagged', 'project': 'prj'}, record.fields)
record = Record.get_by_id('multiple_2')
self.assertEquals(set(['empty', 'fields', 'project=prj']),
set(record.tags))
self.assertEquals({'project': 'prj'}, record.fields)
def test_post_single_key_update(self):
self.mock_current_user(is_admin=True)
response = self.test_app.get('/post',
params={
'key': 'single_key_update',
'fields': '{"project": "prj"}',
})
self.assertEquals('', response.body)
record = Record.get_by_id('single_key_update')
self.assertTrue(record != None)
self.assertEquals(['project=prj'], record.tags)
self.assertEquals({'project': 'prj'}, record.fields)
old_count = Record.query().count()
response = self.test_app.get(
'/post',
params={
'key': 'single_key_update',
'tags': '1,2,3',
'fields':
'{"update": "the", "same": "record", "project": "prj"}',
})
self.assertEquals('', response.body)
self.assertEquals(old_count, Record.query().count())
record = Record.get_by_id('single_key_update')
self.assertTrue(record != None)
self.assertEquals(set(['1', '2', '3', 'project=prj']),
set(record.tags))
self.assertEquals({
'update': 'the',
'same': 'record',
'project': 'prj'
}, record.fields)
def test_post_multiple_empty(self):
self.mock_current_user(is_admin=True)
old_count = Record.query().count()
response = self.test_app.post('/post')
self.assertEquals('', response.body)
self.assertEquals(old_count, Record.query().count())
response = self.test_app.post('/post', params={'p': '{}'})
self.assertEquals('Empty record entries disallowed', response.body)
self.assertEquals(old_count, Record.query().count())
def test_post_multiple_empty(self):
self.mock_current_user(is_admin=True)
old_count = Record.query().count()
response = self.test_app.post('/post')
self.assertEquals('', response.body)
self.assertEquals(old_count, Record.query().count())
response = self.test_app.post('/post', params={'p': '{}'})
self.assertEquals('Empty record entries disallowed', response.body)
self.assertEquals(old_count, Record.query().count())
def test_post_multiple(self):
self.mock_current_user(is_admin=True)
old_count = Record.query().count()
response = self.test_app.post('/post',
params=[
('p',
json.dumps({
'key':
'multiple_0',
'tags': ['hello', 'world'],
'fields': {
'hello': 'world',
'project': 'prj'
},
})),
('p',
json.dumps({
'key': 'multiple_1',
'fields': {
'issue': 'autotagged',
'project': 'prj'
},
})),
('p',
json.dumps({
'key':
'multiple_2',
'tags': ['empty', 'fields'],
'fields': {
'project': 'prj'
},
})),
])
self.assertEquals('', response.body)
self.assertEquals(old_count + 3, Record.query().count())
record = Record.get_by_id('multiple_0')
self.assertEquals(set(['hello', 'world', 'project=prj']),
set(record.tags))
self.assertEquals({'hello': 'world', 'project': 'prj'}, record.fields)
record = Record.get_by_id('multiple_1')
self.assertEquals(['issue=autotagged', 'project=prj'], record.tags)
self.assertEquals({
'issue': 'autotagged',
'project': 'prj'
}, record.fields)
record = Record.get_by_id('multiple_2')
self.assertEquals(set(['empty', 'fields', 'project=prj']),
set(record.tags))
self.assertEquals({'project': 'prj'}, record.fields)
def test_post_multiple_empty(self):
self.mock_current_user(is_admin=False)
password = '******'
Password(id=CQ_BOT_PASSWORD_KEY, sha1=utils.password_sha1(password)).put()
old_count = Record.query().count()
response = self.test_app.post('/post', params={'password': password})
self.assertEquals('', response.body)
self.assertEquals(old_count, Record.query().count())
response = self.test_app.post('/post', params={
'p': '{}', 'password': password})
self.assertEquals('Empty record entries disallowed', response.body)
self.assertEquals(old_count, Record.query().count())
def execute_query(key, begin, end, tags, fields, count, cursor):
records = []
next_cursor = ''
if key and count > 0:
record = Record.get_by_id(key)
if record and ((not begin or record.timestamp >= begin) and
(not end or record.timestamp <= end)
and set(tags).issubset(record.tags)
and matches_fields(fields, record)):
records.append(record)
more = False
else:
more = True
while more and len(records) < count:
filters = []
if begin:
filters.append(Record.timestamp >= begin)
if end:
filters.append(Record.timestamp <= end)
for tag in tags:
filters.append(Record.tags == tag)
query = Record.query().filter(*filters).order(-Record.timestamp)
page_records, next_cursor, more = query.fetch_page(
count - len(records),
start_cursor=Cursor(urlsafe=next_cursor or cursor))
next_cursor = next_cursor.urlsafe() if next_cursor else ''
for record in page_records:
if matches_fields(fields, record):
records.append(record)
return {
'results': [record.to_dict() for record in records],
'cursor': next_cursor,
'more': more,
}
def get_raw_attempts(codereview_hostname, issue, patch):
"""Returns a generator for raw attempts."""
# Do not filter by TAG_CODEREVIEW_HOSTNAME here, because it is not set for old
# issues.
query = Record.query().order(Record.timestamp).filter(
Record.tags == TAG_ISSUE % issue,
Record.tags == TAG_PATCHSET % patch)
raw_attempt = None
count = 0
for record in query:
if not record.matches_codereview_hostname(codereview_hostname):
continue
if raw_attempt is None and TAG_START in record.tags:
raw_attempt = []
if raw_attempt is not None: # pragma: no branch
raw_attempt.append(record)
if TAG_STOP in record.tags:
count += 1
logging.debug('attempt %d has %d records', count, len(raw_attempt))
yield raw_attempt
raw_attempt = None
if raw_attempt: # pragma: no cover
# In cq_stats and Dremel we ignore attempts that do not have patch_stop
# event. However, it may be not a good decision here as this app is
# user-facing and we don't want users to be confused why their last attempt
# is now shown until attempt is actually complete (or if for some reason
# patch_stop message was lost).
count += 1
logging.debug('attempt %d has %d records', count, len(raw_attempt))
yield raw_attempt
def attempts_for_interval(begin, end): # pragma: no cover
finished_in_interval = Record.query().filter(
Record.tags == TAG_STOP,
Record.timestamp >= begin,
Record.timestamp < end)
finish_timestamps = {}
for record in finished_in_interval:
if all(i in record.fields for i in ('project', 'issue', 'patchset')):
key = (
record.fields['project'],
record.fields['issue'],
record.fields['patchset'],
)
finish_timestamps.setdefault(key, []).append(record.timestamp)
for key in finish_timestamps:
# NDB seems to cache records beyond the soft memory limit.
# Force a cache clear between each patchset analysis run to avoid getting
# terminated by Appengine.
ndb.get_context().clear_cache()
last_finish_timestamp = max(finish_timestamps[key])
project, issue, patchset = key
interval_query = Record.query().order(Record.timestamp).filter(
Record.timestamp <= last_finish_timestamp,
Record.tags == TAG_PROJECT % project,
Record.tags == TAG_ISSUE % issue,
Record.tags == TAG_PATCHSET % patchset)
all_attempts = []
interval_attempts = []
attempt = None
for record in interval_query:
if attempt is None and TAG_START in record.tags:
attempt = []
if attempt is not None:
attempt.append(record)
if TAG_STOP in record.tags:
if record.timestamp >= begin:
interval_attempts.append(attempt)
all_attempts.append(attempt)
attempt = None
if len(all_attempts) == 0:
logging.warning('No attempts found for %s issue %s patchset %s at %s' %
(project, issue, patchset, begin))
continue
yield project, issue, patchset, all_attempts, interval_attempts
def attempts_for_interval(begin, end): # pragma: no cover
finished_in_interval = Record.query().filter(Record.tags == TAG_STOP,
Record.timestamp >= begin,
Record.timestamp < end)
finish_timestamps = {}
for record in finished_in_interval:
if all(i in record.fields for i in ('project', 'issue', 'patchset')):
key = (
record.fields['project'],
record.fields['issue'],
record.fields['patchset'],
)
finish_timestamps.setdefault(key, []).append(record.timestamp)
for key in finish_timestamps:
# NDB seems to cache records beyond the soft memory limit.
# Force a cache clear between each patchset analysis run to avoid getting
# terminated by Appengine.
ndb.get_context().clear_cache()
last_finish_timestamp = max(finish_timestamps[key])
project, issue, patchset = key
interval_query = Record.query().order(Record.timestamp).filter(
Record.timestamp <= last_finish_timestamp,
Record.tags == TAG_PROJECT % project,
Record.tags == TAG_ISSUE % issue,
Record.tags == TAG_PATCHSET % patchset)
all_attempts = []
interval_attempts = []
attempt = None
for record in interval_query:
if TAG_START in record.tags:
attempt = []
if attempt != None:
attempt.append(record)
if TAG_STOP in record.tags:
if record.timestamp >= begin:
interval_attempts.append(attempt)
all_attempts.append(attempt)
attempt = None
if len(all_attempts) == 0:
logging.warning(
'No attempts found for %s issue %s patchset %s at %s' %
(project, issue, patchset, begin))
continue
yield project, issue, patchset, all_attempts, interval_attempts
def test_post_multiple_empty(self):
self.mock_current_user(is_admin=False)
password = '******'
Password(id=CQ_BOT_PASSWORD_KEY,
sha1=utils.password_sha1(password)).put()
old_count = Record.query().count()
response = self.test_app.post('/post', params={'password': password})
self.assertEquals('', response.body)
self.assertEquals(old_count, Record.query().count())
response = self.test_app.post('/post',
params={
'p': '{}',
'password': password
})
self.assertEquals('Empty record entries disallowed', response.body)
self.assertEquals(old_count, Record.query().count())
def missing_intervals(minutes, end):
last_cq_stats = CQStats.query().filter(
CQStats.interval_minutes == minutes).order(-CQStats.end).get()
if last_cq_stats:
return intervals_in_range(minutes, last_cq_stats.end, end)
earliest_record = Record.query().order(Record.timestamp).get()
if earliest_record:
begin = earliest_record.timestamp - timedelta(minutes=minutes)
return intervals_in_range(minutes, begin, end)
return []
def _load_records(self, filename):
assert Record.query().count() == 0
records = _load_json(filename)
for record in records:
self.mock_now(datetime.utcfromtimestamp(record['timestamp']))
Record(
id=record['key'],
tags=record['tags'],
fields=record['fields'],
).put()
def missing_intervals(minutes, end):
last_cq_stats = CQStats.query().filter(
CQStats.interval_minutes == minutes).order(-CQStats.end).get()
if last_cq_stats:
return intervals_in_range(minutes, last_cq_stats.end, end)
earliest_record = Record.query().order(Record.timestamp).get()
if earliest_record:
begin = earliest_record.timestamp - timedelta(minutes=minutes)
return intervals_in_range(minutes, begin, end)
return []
def _load_records(self, filename):
assert Record.query().count() == 0
records = _load_json(filename)
for record in records:
self.mock_now(datetime.utcfromtimestamp(record['timestamp']))
Record(
id=record['key'],
tags=record['tags'],
fields=record['fields'],
).put()
def get_raw_attempts(issue, patch): # pragma: no cover
query = Record.query().order(Record.timestamp).filter(
Record.tags == TAG_ISSUE % issue, Record.tags == TAG_PATCHSET % patch)
raw_attempts = []
raw_attempt = None
for record in query:
if raw_attempt == None and TAG_START in record.tags:
raw_attempt = []
if raw_attempt != None:
raw_attempt.append(record)
if TAG_STOP in record.tags:
raw_attempts.append(raw_attempt)
raw_attempt = None
if raw_attempt != None and len(raw_attempt) > 0:
raw_attempts.append(raw_attempt)
return raw_attempts
def get_raw_attempts(issue, patch): # pragma: no cover
query = (
Record.query()
.order(Record.timestamp)
.filter(Record.tags == TAG_ISSUE % issue, Record.tags == TAG_PATCHSET % patch)
)
raw_attempts = []
raw_attempt = None
for record in query:
if raw_attempt == None and TAG_START in record.tags:
raw_attempt = []
if raw_attempt != None:
raw_attempt.append(record)
if TAG_STOP in record.tags:
raw_attempts.append(raw_attempt)
raw_attempt = None
if raw_attempt != None and len(raw_attempt) > 0:
raw_attempts.append(raw_attempt)
return raw_attempts
def get_attempts(issue, patch): # pragma: no cover
"""Given an issue and a patch, returns a list of attempts.
Returns a list of attempts. Attempts are lists of records which fall within
the endpoints of patch_start and patch_stop actions, inclusive.
"""
query = Record.query().order(Record.timestamp).filter(
Record.tags == TAG_ISSUE % issue,
Record.tags == TAG_PATCHSET % patch)
attempt = None
for record in query:
action = record.fields.get('action')
if attempt is None and action == 'patch_start':
attempt = [record]
# Sometimes CQ sends multiple patch_start in a single attempt. These
# are ignored (only the first patch_start is kept).
if attempt is not None and action != 'patch_start':
attempt.append(record)
if action == 'patch_stop':
yield attempt
attempt = None
if attempt != None:
yield attempt
def execute_query(
key, begin, end, tags, fields, count, cursor): # pragma: no cover
records = []
next_cursor = ''
if key and count > 0:
record = Record.get_by_id(key)
if record and (
(not begin or record.timestamp >= begin) and
(not end or record.timestamp <= end) and
set(tags).issubset(record.tags) and
matches_fields(fields, record)):
records.append(record)
more = False
else:
more = True
while more and len(records) < count:
filters = []
if begin:
filters.append(Record.timestamp >= begin)
if end:
filters.append(Record.timestamp <= end)
for tag in tags:
filters.append(Record.tags == tag)
query = Record.query().filter(*filters).order(-Record.timestamp)
page_records, next_cursor, more = query.fetch_page(count - len(records),
start_cursor=Cursor(urlsafe=next_cursor or cursor))
next_cursor = next_cursor.urlsafe() if next_cursor else ''
for record in page_records:
if matches_fields(fields, record):
records.append(record)
return {
'results': [record.to_dict() for record in records],
'cursor': next_cursor,
'more': more,
}
def clear_records():
for record in Record.query():
record.key.delete()
assert Record.query().count() == 0
def test_post_single_empty(self):
self.mock_current_user(is_admin=True)
old_count = Record.query().count()
response = self.test_app.get('/post')
self.assertEquals('Empty record entries disallowed', response.body)
self.assertEquals(old_count, Record.query().count())
def _clear_records(): # pragma: no cover
for record in Record.query():
record.key.delete()
assert Record.query().count() == 0
def test_post_single_empty(self):
self.mock_current_user(is_admin=True)
old_count = Record.query().count()
response = self.test_app.get('/post')
self.assertEquals('Empty record entries disallowed', response.body)
self.assertEquals(old_count, Record.query().count())
def clear_records():
for record in Record.query():
record.key.delete()
assert Record.query().count() == 0
def _clear_records(): # pragma: no cover
for record in Record.query():
record.key.delete()
assert Record.query().count() == 0
