def _clear_ndb(): # pragma: no cover
for cq_stats in CQStats.query():
cq_stats.key.delete()
assert CQStats.query().count() == 0
for record in Record.query():
record.key.delete()
assert Record.query().count() == 0
def get_list(self):
wd = self.app.wd
self.app.open_home_page()
self.rec_cash = []
for element in wd.find_elements_by_xpath(
'//*[@id="maintable"]/tbody/tr[@name="entry"]'):
# dummy record to append cash
record_to_cash = Record()
setattr(
record_to_cash, 'id',
element.find_element_by_name('selected[]').get_attribute(
'value'))
setattr(record_to_cash, 'firstname',
self.record_cell_value(element, 3))
setattr(record_to_cash, 'lastname',
self.record_cell_value(element, 2))
setattr(record_to_cash, 'address',
self.record_cell_value(element, 4))
# synthetic properties for phones and emails
record_to_cash._emails_from_home = self.record_cell_value(
element, 5)
record_to_cash._phones_from_home = self.record_cell_value(
element, 6)
self.rec_cash.append(record_to_cash)
return list(self.rec_cash)
def execute_query(key, begin, end, tags, fields, count, cursor):
records = []
next_cursor = ''
if key and count > 0:
record = Record.get_by_id(key)
if record and ((not begin or record.timestamp >= begin) and
(not end or record.timestamp <= end)
and set(tags).issubset(record.tags)
and matches_fields(fields, record)):
records.append(record)
more = False
else:
more = True
while more and len(records) < count:
filters = []
if begin:
filters.append(Record.timestamp >= begin)
if end:
filters.append(Record.timestamp <= end)
for tag in tags:
filters.append(Record.tags == tag)
query = Record.query().filter(*filters).order(-Record.timestamp)
page_records, next_cursor, more = query.fetch_page(
count - len(records),
start_cursor=Cursor(urlsafe=next_cursor or cursor))
next_cursor = next_cursor.urlsafe() if next_cursor else ''
for record in page_records:
if matches_fields(fields, record):
records.append(record)
return {
'results': [record.to_dict() for record in records],
'cursor': next_cursor,
'more': more,
}
def test_post_multiple(self):
self.mock_current_user(is_admin=True)
old_count = Record.query().count()
response = self.test_app.post('/post', params=[
('p', json.dumps({
'key': 'multiple_0',
'tags': ['hello', 'world'],
'fields': {'hello': 'world', 'project': 'prj'},
})),
('p', json.dumps({
'key': 'multiple_1',
'fields': {'issue': 'autotagged', 'project': 'prj'},
})),
('p', json.dumps({
'key': 'multiple_2',
'tags': ['empty', 'fields'],
'fields': {'project': 'prj'},
})),
])
self.assertEquals('', response.body)
self.assertEquals(old_count + 3, Record.query().count())
record = Record.get_by_id('multiple_0')
self.assertEquals(set(['hello', 'world', 'project=prj']), set(record.tags))
self.assertEquals({'hello': 'world', 'project': 'prj'}, record.fields)
record = Record.get_by_id('multiple_1')
self.assertEquals(['issue=autotagged', 'project=prj'], record.tags)
self.assertEquals({'issue': 'autotagged', 'project': 'prj'}, record.fields)
record = Record.get_by_id('multiple_2')
self.assertEquals(set(['empty', 'fields', 'project=prj']),
set(record.tags))
self.assertEquals({'project': 'prj'}, record.fields)
def test_post_single_key_update(self):
self.mock_current_user(is_admin=True)
response = self.test_app.get('/post',
params={
'key': 'single_key_update',
'fields': '{"project": "prj"}',
})
self.assertEquals('', response.body)
record = Record.get_by_id('single_key_update')
self.assertTrue(record != None)
self.assertEquals(['project=prj'], record.tags)
self.assertEquals({'project': 'prj'}, record.fields)
old_count = Record.query().count()
response = self.test_app.get(
'/post',
params={
'key': 'single_key_update',
'tags': '1,2,3',
'fields':
'{"update": "the", "same": "record", "project": "prj"}',
})
self.assertEquals('', response.body)
self.assertEquals(old_count, Record.query().count())
record = Record.get_by_id('single_key_update')
self.assertTrue(record != None)
self.assertEquals(set(['1', '2', '3', 'project=prj']),
set(record.tags))
self.assertEquals({
'update': 'the',
'same': 'record',
'project': 'prj'
}, record.fields)
def test_query_filtered_key(self):
_clear_records()
Record(id='match', tags=['match_tag']).put()
response = self.test_app.get('/query',
params={
'key': 'match',
'tags': 'match_tag'
})
self.assertEquals(
{
'more':
False,
'results': [{
'key': 'match',
'tags': ['match_tag'],
'fields': {},
}],
}, _parse_body(response))
Record(id='mismatched', tags=['mismatched_tag']).put()
response = self.test_app.get('/query',
params={
'key': 'mismatched',
'tags': 'match_tag'
})
self.assertEquals({
'more': False,
'results': [],
}, _parse_body(response))
def test_query_tags(self):
_clear_records()
Record(id='match', tags=['match_tag_a', 'match_tag_b']).put()
Record(id='match_extra',
tags=['match_tag_a', 'match_tag_b', 'extra_tag']).put()
Record(id='missing_tag', tags=['match_tag_a']).put()
Record(id='wrong_tags', tags=['tag_mismatch_a',
'tag_mismatch_b']).put()
Record(id='no_tags').put()
response = self.test_app.get(
'/query', params={'tags': 'match_tag_a,match_tag_b'})
self.assertEquals(
{
'more':
False,
'results': [{
'key': 'match',
'tags': ['match_tag_a', 'match_tag_b'],
'fields': {},
}, {
'key': 'match_extra',
'tags': ['extra_tag', 'match_tag_a', 'match_tag_b'],
'fields': {},
}],
}, _parse_body(response))
def test_post_single_key_update(self):
self.mock_current_user(is_admin=True)
response = self.test_app.get('/post', params={
'key': 'single_key_update',
'fields': '{"project": "prj"}',
})
self.assertEquals('', response.body)
record = Record.get_by_id('single_key_update')
self.assertTrue(record != None)
self.assertEquals(['project=prj'], record.tags)
self.assertEquals({'project': 'prj'}, record.fields)
old_count = Record.query().count()
response = self.test_app.get('/post', params={
'key': 'single_key_update',
'tags': '1,2,3',
'fields': '{"update": "the", "same": "record", "project": "prj"}',
})
self.assertEquals('', response.body)
self.assertEquals(old_count, Record.query().count())
record = Record.get_by_id('single_key_update')
self.assertTrue(record != None)
self.assertEquals(set(['1', '2', '3', 'project=prj']), set(record.tags))
self.assertEquals({'update': 'the', 'same': 'record', 'project': 'prj'},
record.fields)
def test_missing_intervals_records_only(self):
self.mock_now(datetime(2000, 1, 2, 0))
Record().put()
self.mock_now(datetime(2000, 1, 3, 0))
Record().put()
self.assertEqual([
(datetime(2000, 1, 1, 8), datetime(2000, 1, 2, 8)),
(datetime(2000, 1, 2, 8), datetime(2000, 1, 3, 8)),
], missing_intervals(1440, datetime(2000, 1, 4, 0)))
def _load_records(self, filename):
assert Record.query().count() == 0
records = _load_json(filename)
for record in records:
self.mock_now(datetime.utcfromtimestamp(record['timestamp']))
Record(
id=record['key'],
tags=record['tags'],
fields=record['fields'],
).put()
def test_query_fields(self):
_clear_records()
Record(id='match',
fields={
'match_key_a': 'match_value_a',
'match_key_b': 'match_value_b',
}).put()
Record(id='match_extra',
fields={
'match_key_a': 'match_value_a',
'match_key_b': 'match_value_b',
'extra_key': 'extra_value',
}).put()
Record(id='missing_key',
fields={
'match_key_a': 'match_value_a',
'extra_key': 'extra_value',
}).put()
Record(id='wrong_value',
fields={
'match_key_a': 'match_value_a',
'match_key_b': 'mismatched_value_b',
}).put()
Record(id='empty_fields').put()
response = self.test_app.get('/query',
params={
'fields':
json.dumps({
'match_key_a':
'match_value_a',
'match_key_b':
'match_value_b',
})
})
self.assertEquals(
{
'more':
False,
'results': [{
'key': 'match',
'tags': [],
'fields': {
'match_key_a': 'match_value_a',
'match_key_b': 'match_value_b',
},
}, {
'key': 'match_extra',
'tags': [],
'fields': {
'match_key_a': 'match_value_a',
'match_key_b': 'match_value_b',
'extra_key': 'extra_value',
},
}],
}, _parse_body(response))
def test_post_multiple_empty(self):
self.mock_current_user(is_admin=True)
old_count = Record.query().count()
response = self.test_app.post('/post')
self.assertEquals('', response.body)
self.assertEquals(old_count, Record.query().count())
response = self.test_app.post('/post', params={'p': '{}'})
self.assertEquals('Empty record entries disallowed', response.body)
self.assertEquals(old_count, Record.query().count())
def test_post_multiple_empty(self):
self.mock_current_user(is_admin=True)
old_count = Record.query().count()
response = self.test_app.post('/post')
self.assertEquals('', response.body)
self.assertEquals(old_count, Record.query().count())
response = self.test_app.post('/post', params={'p': '{}'})
self.assertEquals('Empty record entries disallowed', response.body)
self.assertEquals(old_count, Record.query().count())
def test_modify_first_record(app):
app.record.provide()
old_rec = app.record.get_list()
record = Record(firstname='modified_again', lastname='modified')
record.id = old_rec[0].id
app.record.modify_first(record)
assert len(old_rec) == app.record.count()
new_rec = app.record.get_list()
old_rec[0] = record
assert sorted(old_rec,
key=Record.id_or_max) == sorted(new_rec,
key=Record.id_or_max)
def test_post_multiple(self):
self.mock_current_user(is_admin=True)
old_count = Record.query().count()
response = self.test_app.post('/post',
params=[
('p',
json.dumps({
'key':
'multiple_0',
'tags': ['hello', 'world'],
'fields': {
'hello': 'world',
'project': 'prj'
},
})),
('p',
json.dumps({
'key': 'multiple_1',
'fields': {
'issue': 'autotagged',
'project': 'prj'
},
})),
('p',
json.dumps({
'key':
'multiple_2',
'tags': ['empty', 'fields'],
'fields': {
'project': 'prj'
},
})),
])
self.assertEquals('', response.body)
self.assertEquals(old_count + 3, Record.query().count())
record = Record.get_by_id('multiple_0')
self.assertEquals(set(['hello', 'world', 'project=prj']),
set(record.tags))
self.assertEquals({'hello': 'world', 'project': 'prj'}, record.fields)
record = Record.get_by_id('multiple_1')
self.assertEquals(['issue=autotagged', 'project=prj'], record.tags)
self.assertEquals({
'issue': 'autotagged',
'project': 'prj'
}, record.fields)
record = Record.get_by_id('multiple_2')
self.assertEquals(set(['empty', 'fields', 'project=prj']),
set(record.tags))
self.assertEquals({'project': 'prj'}, record.fields)
def update_record(key=None, tags=None, fields=None): # pragma: no cover
tags = tags or []
fields = fields or {}
if not key and len(tags) == 0 and len(fields) == 0:
raise ValueError('Empty record entries disallowed')
if not 'project' in fields:
raise ValueError('"Project" field missing')
for item in fields:
if item in AUTO_TAGGED_FIELDS:
tags.append('%s=%s' % (item, fields[item]))
record = Record(id=key)
record.tags = list(set(tags))
record.fields = fields
record.put()
def test_post_multiple_empty(self):
self.mock_current_user(is_admin=False)
password = '******'
Password(id=CQ_BOT_PASSWORD_KEY, sha1=utils.password_sha1(password)).put()
old_count = Record.query().count()
response = self.test_app.post('/post', params={'password': password})
self.assertEquals('', response.body)
self.assertEquals(old_count, Record.query().count())
response = self.test_app.post('/post', params={
'p': '{}', 'password': password})
self.assertEquals('Empty record entries disallowed', response.body)
self.assertEquals(old_count, Record.query().count())
def test_modify_random_record(app, db, check_ui):
app.record.provide()
old_rec = db.get_record_list()
record = choice(old_rec)
donor = Record(firstname='modified_again', lastname='modified')
donor.id = record.id
app.record.modify_by_id(upd_record=donor, rec_id=record.id)
new_rec = db.get_record_list()
assert len(old_rec) == len(new_rec)
old_rec[old_rec.index(record)] = donor
assert old_rec == new_rec
if check_ui:
assert sorted(app.record.get_list(),
key=Record.id_or_max) == sorted(new_rec,
key=Record.id_or_max)
def get_record_list(self, extended=False):
record_list = []
cursor = self.connection.cursor()
try:
if not extended:
cursor.execute(
'select id, firstname, lastname from addressbook where deprecated="0000-00-00 00:00:00"'
)
for row in cursor:
(id, firstname, lastname) = row
record_list.append(
Record(id=str(id),
firstname=firstname,
lastname=lastname))
elif extended:
cursor.execute('select id, firstname, lastname, \
address, home, mobile, work, phone2, email, email2, email3 \
from addressbook \
where deprecated="0000-00-00 00:00:00"')
for row in cursor:
(
id,
firstname,
lastname,
address,
home,
mobile,
work,
phone2,
email,
email2,
email3,
) = row
record_list.append(
Record(id=str(id),
firstname=firstname,
lastname=lastname,
address=address,
home=home,
mobile=mobile,
work=work,
phone2=phone2,
email=email,
email2=email2,
email3=email3))
finally:
cursor.close()
return record_list
def test_query_empty(self):
_clear_records()
response = self.test_app.get('/query')
self.assertEquals({
'more': False,
'results': [],
'cursor': '',
}, json.loads(response.body))
Record(id='test_key', tags=['a', 'b', 'c'], fields={
'test': 'field'
}).put()
response = self.test_app.get('/query')
self.assertEquals(
{
'more':
False,
'results': [{
'key': 'test_key',
'tags': ['a', 'b', 'c'],
'fields': {
'test': 'field'
},
}],
}, _parse_body(response))
def get(self, user_id: str, record_date: str) -> Record:
if not record_date:
raise MissingInputException("record_date")
if not user_id:
raise MissingInputException("user_id")
key = {"user": {"S": user_id}, "record_date": {"S": record_date}}
item = self.dynamo.get_item(table=self.records_table_name, key=key)
if not item:
raise RecordNotExistsException()
return Record(
record_date=item["record_date"]["S"],
perimeter_arm=item["perimeter_arm"]["N"],
perimeter_cut=item["perimeter_cut"]["N"],
perimeter_belly=item["perimeter_belly"]["N"],
perimeter_leg=item["perimeter_leg"]["N"],
perimeter_hips=item["perimeter_hips"]["N"],
intakes=list(
map(
lambda i: Intake(description=i["description"]["S"],
calories=i["calories"]["N"]),
item["intakes"]["L"])),
intake_score=item["intake_score"]["N"],
sports=list(
map(lambda i: Sport(description=i["description"]["S"]),
item["intakes"]["L"])),
sports_score=item["sports_score"]["N"])
def test_query_end(self):
_clear_records()
self.mock_now(datetime.utcfromtimestamp(0))
Record(id='match').put()
self.mock_now(datetime.utcfromtimestamp(10))
Record(id='too_late').put()
response = self.test_app.get('/query', params={'end': 5})
self.assertEquals(
{
'more': False,
'results': [{
'key': 'match',
'tags': [],
'fields': {},
}],
}, _parse_body(response))
def test_post_single_auto_tagged(self):
self.mock_current_user(is_admin=True)
response = self.test_app.get(
'/post',
params={
'key':
'single_auto_tagged',
'tags':
'existingTag,issue=hello',
'fields':
'{"issue": "hello", "patchset": "world", "project": "prj"}',
})
self.assertEquals('', response.body)
record = Record.get_by_id('single_auto_tagged')
self.assertTrue(record != None)
self.assertEquals(
set([
'existingTag', 'issue=hello', 'patchset=world', 'project=prj'
]), set(record.tags))
self.assertEquals(
{
'issue': 'hello',
'patchset': 'world',
'project': 'prj'
}, record.fields)
def get_raw_attempts(codereview_hostname, issue, patch):
"""Returns a generator for raw attempts."""
# Do not filter by TAG_CODEREVIEW_HOSTNAME here, because it is not set for old
# issues.
query = Record.query().order(Record.timestamp).filter(
Record.tags == TAG_ISSUE % issue,
Record.tags == TAG_PATCHSET % patch)
raw_attempt = None
count = 0
for record in query:
if not record.matches_codereview_hostname(codereview_hostname):
continue
if raw_attempt is None and TAG_START in record.tags:
raw_attempt = []
if raw_attempt is not None: # pragma: no branch
raw_attempt.append(record)
if TAG_STOP in record.tags:
count += 1
logging.debug('attempt %d has %d records', count, len(raw_attempt))
yield raw_attempt
raw_attempt = None
if raw_attempt: # pragma: no cover
# In cq_stats and Dremel we ignore attempts that do not have patch_stop
# event. However, it may be not a good decision here as this app is
# user-facing and we don't want users to be confused why their last attempt
# is now shown until attempt is actually complete (or if for some reason
# patch_stop message was lost).
count += 1
logging.debug('attempt %d has %d records', count, len(raw_attempt))
yield raw_attempt
def test_compare_entire_home_page_with_db_data(app, db):
# provide 3 records
app.record.provide(
requested=3,
record=Record(
firstname='John', # i use one word for names
lastname='Smith',
address='111 Jeegurda street apt 234 Another YE 00000 DFR',
home='+11111111111', # mobile has been intentionally not set
work='1 333 333 3333',
phone2='1-444-444-4444',
email='*****@*****.**', # email 2 has been intentionally not set
email3='*****@*****.**'))
# get whole list from home page
hp_list = sorted(app.record.get_list(), key=Record.id_or_max)
# get whole not deprecated list from db
db_list = db.get_record_list(extended=True)
# compare lists record by record (sort by id) starting from lists lengths
assert len(hp_list) == len(db_list)
# if equal - continue: for each in list
for i in range(len(hp_list)):
# compare ids and names just comparing records representations
assert hp_list[i] == db_list[i]
# compare address
assert hp_list[i].address == db_list[i].address
# compare phones in homepage format
assert hp_list[i]._phones_from_home == homepage_like_set(
db_list[i], set_to_arrange=db_list[i]._phones)
# verify emails as well
assert hp_list[i]._emails_from_home == homepage_like_set(
db_list[i], set_to_arrange=db_list[i]._emails)
def test_compare_homepage_properties_with_edit_page_values_for_random_record(
app):
# provide record if there is no record on the page
app.record.provide(record=Record(
firstname='John', # i use one word for names
lastname='Smith',
address='111 Jeegurda street apt 234 Another YE 00000 DFR',
home='+11111111111', # mobile has been intentionally not set
work='1 333 333 3333',
phone2='1-444-444-4444',
email='*****@*****.**', # email 2 has been intentionally not set
email3='*****@*****.**'))
# set random index to take record from list of existing records
record_index = randrange(app.record.count())
# get text properties from home page table for random record
record_from_home = app.record.get_list()[record_index]
# get all the fields from edit page for the record selected (names and phones included as requested)
record_from_edit = app.record.get_info_from_edit(record_index,
htmlized=True)
# assert names
assert record_from_home.firstname == record_from_edit.firstname
assert record_from_home.lastname == record_from_edit.lastname
# assert address
assert record_from_home.address == record_from_edit.address
# assert phones (backward check here)
assert record_from_home._phones_from_home == homepage_like_set(
record_from_edit, set_to_arrange=record_from_edit._phones)
# assert emails (backward check here)
assert record_from_home._emails_from_home == homepage_like_set(
record_from_edit, set_to_arrange=record_from_edit._emails)
def main(event, context):
user_id = event.get("pathParameters", {}).get("user_id")
record = Record(**json.loads(event["body"]))
RecordRepository().save(user_id, record)
return {"statusCode": 200, "body": json.dumps({"status": "Created"})}
def get_info_from_edit(self, index, htmlized=False):
# wd = self.app.wd
self.app.open_home_page()
# create dummy record as source of fields - any property is None
record = Record()
self.open_edit_by_index(index=index)
# read the form and return the 'Record' object updated
return self.read_form(record, htmlized)
def attempts_for_interval(begin, end): # pragma: no cover
finished_in_interval = Record.query().filter(Record.tags == TAG_STOP,
Record.timestamp >= begin,
Record.timestamp < end)
finish_timestamps = {}
for record in finished_in_interval:
if all(i in record.fields for i in ('project', 'issue', 'patchset')):
key = (
record.fields['project'],
record.fields['issue'],
record.fields['patchset'],
)
finish_timestamps.setdefault(key, []).append(record.timestamp)
for key in finish_timestamps:
# NDB seems to cache records beyond the soft memory limit.
# Force a cache clear between each patchset analysis run to avoid getting
# terminated by Appengine.
ndb.get_context().clear_cache()
last_finish_timestamp = max(finish_timestamps[key])
project, issue, patchset = key
interval_query = Record.query().order(Record.timestamp).filter(
Record.timestamp <= last_finish_timestamp,
Record.tags == TAG_PROJECT % project,
Record.tags == TAG_ISSUE % issue,
Record.tags == TAG_PATCHSET % patchset)
all_attempts = []
interval_attempts = []
attempt = None
for record in interval_query:
if TAG_START in record.tags:
attempt = []
if attempt != None:
attempt.append(record)
if TAG_STOP in record.tags:
if record.timestamp >= begin:
interval_attempts.append(attempt)
all_attempts.append(attempt)
attempt = None
if len(all_attempts) == 0:
logging.warning(
'No attempts found for %s issue %s patchset %s at %s' %
(project, issue, patchset, begin))
continue
yield project, issue, patchset, all_attempts, interval_attempts
def attempts_for_interval(begin, end): # pragma: no cover
finished_in_interval = Record.query().filter(
Record.tags == TAG_STOP,
Record.timestamp >= begin,
Record.timestamp < end)
finish_timestamps = {}
for record in finished_in_interval:
if all(i in record.fields for i in ('project', 'issue', 'patchset')):
key = (
record.fields['project'],
record.fields['issue'],
record.fields['patchset'],
)
finish_timestamps.setdefault(key, []).append(record.timestamp)
for key in finish_timestamps:
# NDB seems to cache records beyond the soft memory limit.
# Force a cache clear between each patchset analysis run to avoid getting
# terminated by Appengine.
ndb.get_context().clear_cache()
last_finish_timestamp = max(finish_timestamps[key])
project, issue, patchset = key
interval_query = Record.query().order(Record.timestamp).filter(
Record.timestamp <= last_finish_timestamp,
Record.tags == TAG_PROJECT % project,
Record.tags == TAG_ISSUE % issue,
Record.tags == TAG_PATCHSET % patchset)
all_attempts = []
interval_attempts = []
attempt = None
for record in interval_query:
if attempt is None and TAG_START in record.tags:
attempt = []
if attempt is not None:
attempt.append(record)
if TAG_STOP in record.tags:
if record.timestamp >= begin:
interval_attempts.append(attempt)
all_attempts.append(attempt)
attempt = None
if len(all_attempts) == 0:
logging.warning('No attempts found for %s issue %s patchset %s at %s' %
(project, issue, patchset, begin))
continue
yield project, issue, patchset, all_attempts, interval_attempts
def test_post_multiple_empty(self):
self.mock_current_user(is_admin=False)
password = '******'
Password(id=CQ_BOT_PASSWORD_KEY,
sha1=utils.password_sha1(password)).put()
old_count = Record.query().count()
response = self.test_app.post('/post', params={'password': password})
self.assertEquals('', response.body)
self.assertEquals(old_count, Record.query().count())
response = self.test_app.post('/post',
params={
'p': '{}',
'password': password
})
self.assertEquals('Empty record entries disallowed', response.body)
self.assertEquals(old_count, Record.query().count())
def test_post_project_missing(self):
self.mock_current_user(is_admin=True)
response = self.test_app.get('/post', params={
'key': 'single_packet',
'fields': '{"some": "random"}',
})
self.assertEquals('"Project" field missing', response.body)
record = Record.get_by_id('single_packet')
self.assertIsNone(record)
def test_post_project_missing(self):
self.mock_current_user(is_admin=True)
response = self.test_app.get('/post',
params={
'key': 'single_packet',
'fields': '{"some": "random"}',
})
self.assertEquals('"Project" field missing', response.body)
record = Record.get_by_id('single_packet')
self.assertIsNone(record)
def _load_records(self, filename):
assert Record.query().count() == 0
records = _load_json(filename)
for record in records:
self.mock_now(datetime.utcfromtimestamp(record['timestamp']))
Record(
id=record['key'],
tags=record['tags'],
fields=record['fields'],
).put()
def missing_intervals(minutes, end):
last_cq_stats = CQStats.query().filter(
CQStats.interval_minutes == minutes).order(-CQStats.end).get()
if last_cq_stats:
return intervals_in_range(minutes, last_cq_stats.end, end)
earliest_record = Record.query().order(Record.timestamp).get()
if earliest_record:
begin = earliest_record.timestamp - timedelta(minutes=minutes)
return intervals_in_range(minutes, begin, end)
return []
def missing_intervals(minutes, end):
last_cq_stats = CQStats.query().filter(
CQStats.interval_minutes == minutes).order(-CQStats.end).get()
if last_cq_stats:
return intervals_in_range(minutes, last_cq_stats.end, end)
earliest_record = Record.query().order(Record.timestamp).get()
if earliest_record:
begin = earliest_record.timestamp - timedelta(minutes=minutes)
return intervals_in_range(minutes, begin, end)
return []
def add_record(self, hours_from_start, tagged_fields):
tagged_fields.setdefault('project', 'test')
if tagged_fields.get('action', '').startswith('verifier_'):
tagged_fields.setdefault('verifier', TRYJOBVERIFIER)
self.mock_now(
datetime.utcfromtimestamp(STATS_START_TIMESTAMP) +
timedelta(hours=hours_from_start))
Record(
tags=['%s=%s' % (k, v) for k, v in tagged_fields.items()],
fields=tagged_fields,
).put()
def test_missing_intervals_mismatched_cq_stats(self):
CQStats(project='',
interval_minutes=60,
begin=datetime(2000, 1, 3, 7),
end=datetime(2000, 1, 3, 8)).put()
self.mock_now(datetime(2000, 1, 2, 0))
Record().put()
self.assertEqual([
(datetime(2000, 1, 1, 8), datetime(2000, 1, 2, 8)),
(datetime(2000, 1, 2, 8), datetime(2000, 1, 3, 8)),
], missing_intervals(1440, datetime(2000, 1, 4, 0)))
def test_query_cursor(self):
_clear_records()
for _ in range(10):
Record().put()
response = self.test_app.get('/query', params={'count': 4})
packet = _parse_body(response, preserve_cursor=True)
cursor = packet['cursor']
self.assertEquals(
{
'more': True,
'cursor': cursor,
'results': [{
'key': None,
'tags': [],
'fields': {}
}] * 4,
}, packet)
response = self.test_app.get('/query',
params={
'cursor': cursor,
'count': 4
})
packet = _parse_body(response, preserve_cursor=True)
cursor = packet['cursor']
self.assertEquals(
{
'more': True,
'cursor': cursor,
'results': [{
'key': None,
'tags': [],
'fields': {}
}] * 4,
}, packet)
response = self.test_app.get('/query',
params={
'cursor': cursor,
'count': 4
})
packet = _parse_body(response, preserve_cursor=True)
cursor = packet['cursor']
self.assertEquals(
{
'more': False,
'cursor': cursor,
'results': [{
'key': None,
'tags': [],
'fields': {}
}] * 2,
}, packet)
def test_post_single_packet(self):
self.mock_current_user(is_admin=True)
response = self.test_app.get('/post', params={
'key': 'single_packet',
'tags': 'tagA,tagB,tagC',
'fields': '{"some": "random", "json": ["data"], "project": "prj"}',
})
self.assertEquals('', response.body)
record = Record.get_by_id('single_packet')
self.assertTrue(record != None)
self.assertEquals(set(['tagA', 'tagB', 'tagC', 'project=prj']),
set(record.tags))
self.assertEquals({'some': 'random', 'json': ['data'], 'project': 'prj'},
record.fields)
def execute_query(
key, begin, end, tags, fields, count, cursor): # pragma: no cover
records = []
next_cursor = ''
if key and count > 0:
record = Record.get_by_id(key)
if record and (
(not begin or record.timestamp >= begin) and
(not end or record.timestamp <= end) and
set(tags).issubset(record.tags) and
matches_fields(fields, record)):
records.append(record)
more = False
else:
more = True
while more and len(records) < count:
filters = []
if begin:
filters.append(Record.timestamp >= begin)
if end:
filters.append(Record.timestamp <= end)
for tag in tags:
filters.append(Record.tags == tag)
query = Record.query().filter(*filters).order(-Record.timestamp)
page_records, next_cursor, more = query.fetch_page(count - len(records),
start_cursor=Cursor(urlsafe=next_cursor or cursor))
next_cursor = next_cursor.urlsafe() if next_cursor else ''
for record in page_records:
if matches_fields(fields, record):
records.append(record)
return {
'results': [record.to_dict() for record in records],
'cursor': next_cursor,
'more': more,
}
def test_post_single_auto_tagged(self):
self.mock_current_user(is_admin=True)
response = self.test_app.get('/post', params={
'key': 'single_auto_tagged',
'tags': 'existingTag,issue=hello',
'fields': '{"issue": "hello", "patchset": "world", "project": "prj"}',
})
self.assertEquals('', response.body)
record = Record.get_by_id('single_auto_tagged')
self.assertTrue(record != None)
self.assertEquals(
set(['existingTag', 'issue=hello', 'patchset=world', 'project=prj']),
set(record.tags))
self.assertEquals(
{'issue': 'hello', 'patchset': 'world', 'project': 'prj'},
record.fields)
def get_raw_attempts(issue, patch): # pragma: no cover
query = (
Record.query()
.order(Record.timestamp)
.filter(Record.tags == TAG_ISSUE % issue, Record.tags == TAG_PATCHSET % patch)
)
raw_attempts = []
raw_attempt = None
for record in query:
if raw_attempt == None and TAG_START in record.tags:
raw_attempt = []
if raw_attempt != None:
raw_attempt.append(record)
if TAG_STOP in record.tags:
raw_attempts.append(raw_attempt)
raw_attempt = None
if raw_attempt != None and len(raw_attempt) > 0:
raw_attempts.append(raw_attempt)
return raw_attempts
def get_attempts(issue, patch): # pragma: no cover
"""Given an issue and a patch, returns a list of attempts.
Returns a list of attempts. Attempts are lists of records which fall within
the endpoints of patch_start and patch_stop actions, inclusive.
"""
query = Record.query().order(Record.timestamp).filter(
Record.tags == TAG_ISSUE % issue,
Record.tags == TAG_PATCHSET % patch)
attempt = None
for record in query:
action = record.fields.get('action')
if attempt is None and action == 'patch_start':
attempt = [record]
# Sometimes CQ sends multiple patch_start in a single attempt. These
# are ignored (only the first patch_start is kept).
if attempt is not None and action != 'patch_start':
attempt.append(record)
if action == 'patch_stop':
yield attempt
attempt = None
if attempt != None:
yield attempt
def _clear_records(): # pragma: no cover
for record in Record.query():
record.key.delete()
assert Record.query().count() == 0
def clear_records():
for record in Record.query():
record.key.delete()
assert Record.query().count() == 0
def test_post_single_empty(self):
self.mock_current_user(is_admin=True)
old_count = Record.query().count()
response = self.test_app.get('/post')
self.assertEquals('Empty record entries disallowed', response.body)
self.assertEquals(old_count, Record.query().count())
