def test_fields_from_query(self):
User.delete().execute()
for i in range(3):
user = User.create(username='******' % i)
for x in range(i + 1):
Note.create(user=user, text='%s-%s' % (user.username, x))
query = (User
.select(User.username, fn.COUNT(Note.id).alias('ct'))
.join(Note, JOIN.LEFT_OUTER)
.group_by(User.username)
.order_by(User.id))
with assert_query_count(1):
u0, u1, u2 = list(query)
self.assertEqual(model_to_dict(u0, fields_from_query=query), {
'username': '******',
'ct': 1})
self.assertEqual(model_to_dict(u2, fields_from_query=query), {
'username': '******',
'ct': 3})
notes = (Note
.select(Note, User, SQL('1337').alias('magic'))
.join(User)
.order_by(Note.id)
.limit(1))
with assert_query_count(1):
n1, = notes
res = model_to_dict(n1, fields_from_query=notes)
self.assertEqual(res, {
'id': n1.id,
'magic': 1337,
'text': 'u0-0',
'user': {
'id': n1.user_id,
'username': '******',
},
})
res = model_to_dict(
n1,
fields_from_query=notes,
exclude=[User.id, Note.id])
self.assertEqual(res, {
'magic': 1337,
'text': 'u0-0',
'user': {'username': '******'},
})
# `only` has no effect when using `fields_from_query`.
res = model_to_dict(
n1,
fields_from_query=notes,
only=[User.username])
self.assertEqual(res, {
'id': n1.id,
'magic': 1337,
'text': 'u0-0',
'user': {'id': n1.user_id, 'username': '******'},
})
def test_simple_backref(self):
with assert_query_count(1):
self.assertEqual(model_to_dict(self.user, backrefs=True), {
'id': self.user.id,
'notes': [],
'username': self.user.username})
# Create a note to populate backrefs list.
note = Note.create(user=self.user, text='note-1')
expected = {
'id': self.user.id,
'notes': [
{'id': note.id, 'notetag_set': [], 'text': note.text},
],
'username': self.user.username}
# Two queries: one to get related notes, one to get related notetags.
with assert_query_count(2):
self.assertEqual(
model_to_dict(self.user, backrefs=True),
expected)
query = (User
.select(User, Note, NoteTag)
.join(Note, JOIN_LEFT_OUTER)
.join(NoteTag, JOIN_LEFT_OUTER)
.aggregate_rows())
user = query.get()
with assert_query_count(0):
self.assertEqual(model_to_dict(user, backrefs=True), expected)
def test_extra_attrs(self):
with assert_query_count(0):
extra = ['name_hash', 'title']
self.assertEqual(model_to_dict(self.user, extra_attrs=extra), {
'id': self.user.id,
'username': self.user.username,
'name_hash': 5,
'title': 'Peewee',
})
with assert_query_count(0):
# Unknown attr causes AttributeError.
def fails():
model_to_dict(self.user, extra_attrs=['xx'])
self.assertRaises(AttributeError, fails)
def test_many_to_many(self):
note = Note.create(user=self.user, text='note-1')
t1 = Tag.create(tag='t1')
t2 = Tag.create(tag='t2')
Tag.create(tag='tx') # Note used on any notes.
nt1 = NoteTag.create(note=note, tag=t1)
nt2 = NoteTag.create(note=note, tag=t2)
expected = {
'id': self.user.id,
'notes': [{
'id': note.id,
'notetag_set': [
{'id': nt1.id, 'tag': {'id': t1.id, 'tag': t1.tag}},
{'id': nt2.id, 'tag': {'id': t2.id, 'tag': t2.tag}},
],
'text': note.text,
}],
'username': self.user.username,
}
# Query to retrieve notes, note-tags, and 2 tag queries.
with assert_query_count(4):
self.assertEqual(
model_to_dict(self.user, backrefs=True),
expected)
def test_recursive_fk(self):
root = Category.create(name='root')
child = Category.create(name='child', parent=root)
grandchild = Category.create(name='grandchild', parent=child)
with assert_query_count(0):
self.assertEqual(model_to_dict(root), {
'id': root.id,
'name': root.name,
'parent': {},
})
with assert_query_count(0):
self.assertEqual(model_to_dict(root, recurse=False), {
'id': root.id,
'name': root.name,
'parent': None,
})
with assert_query_count(1):
self.assertEqual(model_to_dict(root, backrefs=True), {
'children': [{'id': child.id, 'name': child.name}],
'id': root.id,
'name': root.name,
'parent': {},
})
with assert_query_count(1):
self.assertEqual(model_to_dict(child, backrefs=True), {
'children': [{'id': grandchild.id, 'name': grandchild.name}],
'id': child.id,
'name': child.name,
'parent': {
'id': root.id,
'name': root.name,
},
})
with assert_query_count(0):
self.assertEqual(model_to_dict(child, backrefs=False), {
'id': child.id,
'name': child.name,
'parent': {
'id': root.id,
'name': root.name,
},
})
def test_simple_recurse(self):
note = Note.create(user=self.user, text='note-1')
with assert_query_count(0):
self.assertEqual(model_to_dict(note), {
'id': note.id,
'text': note.text,
'user': {
'id': self.user.id,
'username': self.user.username}})
with assert_query_count(0):
self.assertEqual(model_to_dict(note, recurse=False), {
'id': note.id,
'text': note.text,
'user': self.user.id,
})
def test_assert_query_count_ctx_mgr(self):
with assert_query_count(3):
for i in range(3):
Data.create(key=str(i))
def will_fail():
with assert_query_count(2):
Data.create(key='x')
self.assertRaises(AssertionError, will_fail)
def test_search_queries(self):
self.populate()
with assert_query_count(6):
results = self.search(
'default',
'testing',
k1='k1-1',
k2='k2-1',
k3='k3-1')
self.assertEqual(results['page'], 1)
self.assertEqual(results['pages'], 5)
self.assertEqual([d['metadata']['k1'] for d in results['documents']],
['k1-1'] * 10)
def test_recurse_backrefs(self):
note = Note.create(user=self.user, text='note-1')
# One query to retrieve the note-tag set.
with assert_query_count(1):
self.assertEqual(model_to_dict(note, backrefs=True), {
'id': note.id,
'notetag_set': [],
'text': note.text,
'user': {
'id': self.user.id,
'username': self.user.username,
},
})
def test_related(self):
data = {
'id': 2,
'text': 'note-1',
'user': {
'id': self.user.id,
'username': '******'}}
with assert_query_count(0):
inst = dict_to_model(Note, data)
self.assertTrue(isinstance(inst, Note))
self.assertEqual(inst.id, 2)
self.assertEqual(inst.text, 'note-1')
self.assertTrue(isinstance(inst.user, User))
self.assertEqual(inst.user.id, self.user.id)
self.assertEqual(inst.user.username, 'charlie')
data['user'] = self.user.id
with assert_query_count(0):
inst = dict_to_model(Note, data)
with assert_query_count(1):
self.assertEqual(inst.user, self.user)
def test_query_count(self):
idx_a = Index.create(name='idx-a')
idx_b = Index.create(name='idx-b')
phrases = ['foo', 'bar', 'baze', 'nug', 'nuggie']
for phrase in phrases:
phrase = 'document ' + phrase
doc = idx_a.index(phrase)
idx_b.index(phrase, doc, foo='bar', baze='nug')
for idx in ['idx-a', 'idx-b']:
for query in ['nug', 'nug*', 'document', 'missing']:
with assert_query_count(6):
# 1. Get index.
# 2. Prefetch indexes.
# 3. Prefetch index documents.
# 4. Prefetch metadata
# 5. Fetch documents (top of prefetch).
# 6. COUNT(*) for pagination.
self.search(idx, query)
with assert_query_count(6):
self.search(idx, query, foo='bar')
with assert_query_count(6):
# Same as above.
self.app.get('/idx-a/')
with assert_query_count(5):
# Same as above minus first query for index.
self.app.get('/documents/')
for i in range(10):
Index.create(name='idx-%s' % i)
with assert_query_count(1):
self.app.get('/')
def test_backrefs(self):
data = {
'id': self.user.id,
'username': '******',
'notes': [
{'id': 1, 'text': 'note-1'},
{'id': 2, 'text': 'note-2'},
]}
with assert_query_count(0):
inst = dict_to_model(User, data)
self.assertEqual(inst.id, self.user.id)
self.assertEqual(inst.username, 'charlie')
self.assertTrue(isinstance(inst.notes, list))
note_1, note_2 = inst.notes
self.assertEqual(note_1.id, 1)
self.assertEqual(note_1.text, 'note-1')
self.assertEqual(note_1.user, self.user)
self.assertEqual(note_2.id, 2)
self.assertEqual(note_2.text, 'note-2')
self.assertEqual(note_2.user, self.user)
def test_list_alive_tags(initialized_db):
found = False
for tag in filter_to_visible_tags(filter_to_alive_tags(Tag.select())):
tags = list_alive_tags(tag.repository)
assert tag in tags
with assert_query_count(1):
legacy_images = get_legacy_images_for_tags(tags)
for tag in tags:
assert ManifestLegacyImage.get(
manifest=tag.manifest).image == legacy_images[tag.id]
found = True
assert found
# Ensure hidden tags cannot be listed.
tag = Tag.get()
tag.hidden = True
tag.save()
tags = list_alive_tags(tag.repository)
assert tag not in tags
def test_get_most_recent_tag_empty_repo(initialized_db):
empty_repo = create_repository("devtable", "empty", None)
with assert_query_count(1):
assert get_most_recent_tag(empty_repo) is None
def test_get_most_recent_tag(initialized_db):
repo = get_repository("outsideorg", "coolrepo")
with assert_query_count(1):
assert get_most_recent_tag(repo).name == "latest"
def test_get_expired_tag(namespace_name, repo_name, tag_name, expected,
initialized_db):
repo = get_repository(namespace_name, repo_name)
with assert_query_count(1):
assert bool(get_expired_tag(repo, tag_name)) == expected
def will_fail():
with assert_query_count(2):
Data.create(key='x')
def test_search_query_count(query, client):
with client_with_identity('devtable', client) as cl:
params = {'query': query}
with assert_query_count(10):
result = conduct_api_call(cl, ConductSearch, 'GET', params, None, 200).json
assert len(result['results'])
def test_simple(self):
with assert_query_count(0):
self.assertEqual(model_to_dict(self.user), {
'id': self.user.id,
'username': self.user.username})
def test_search_query_count(query, client):
with client_with_identity("devtable", client) as cl:
params = {"query": query}
with assert_query_count(10):
result = conduct_api_call(cl, ConductSearch, "GET", params, None, 200).json
assert len(result["results"])
def test_find_garbage_policy_functions(default_tag_policy, initialized_db):
with assert_query_count(1):
one_policy = model.repository.get_random_gc_policy()
all_policies = model.repository._get_gc_expiration_policies()
assert one_policy in all_policies
def test_get_most_recent_tag(initialized_db):
repo = get_repository('outsideorg', 'coolrepo')
with assert_query_count(1):
assert get_most_recent_tag(repo).name == 'latest'
def test_fields_from_query(self):
User.delete().execute()
for i in range(3):
user = User.create(username='******' % i)
for x in range(i + 1):
Note.create(user=user, text='%s-%s' % (user.username, x))
query = (User.select(User.username,
fn.COUNT(Note.id).alias('ct')).join(
Note, JOIN.LEFT_OUTER).group_by(
User.username).order_by(User.id))
with assert_query_count(1):
u0, u1, u2 = list(query)
self.assertEqual(model_to_dict(u0, fields_from_query=query), {
'username': '******',
'ct': 1
})
self.assertEqual(model_to_dict(u2, fields_from_query=query), {
'username': '******',
'ct': 3
})
notes = (Note.select(Note, User,
SQL('1337').alias('magic')).join(User).order_by(
Note.id).limit(1))
with assert_query_count(1):
n1, = notes
res = model_to_dict(n1, fields_from_query=notes)
self.assertEqual(
res, {
'id': n1.id,
'magic': 1337,
'text': 'u0-0',
'user': {
'id': n1.user_id,
'username': '******',
},
})
res = model_to_dict(n1,
fields_from_query=notes,
exclude=[User.id, Note.id])
self.assertEqual(res, {
'magic': 1337,
'text': 'u0-0',
'user': {
'username': '******'
},
})
# `only` has no effect when using `fields_from_query`.
res = model_to_dict(n1,
fields_from_query=notes,
only=[User.username])
self.assertEqual(
res, {
'id': n1.id,
'magic': 1337,
'text': 'u0-0',
'user': {
'id': n1.user_id,
'username': '******'
},
})
def test_simple(self):
with assert_query_count(0):
self.assertEqual(model_to_dict(self.user), {
'id': self.user.id,
'username': self.user.username
})
def will_fail():
with assert_query_count(2):
Data.create(key='x')
def test_index_document_attachments(self):
idx_a = Index.create(name='idx-a')
json_data = json.dumps({
'content': 'doc a',
'index': 'idx-a',
'metadata': {
'k1': 'v1-a',
'k2': 'v2-a'
},
})
response = self.app.post('/documents/',
data={
'data': json_data,
'file_0':
(BytesIO(b'testfile1'), 'test1.txt'),
'file_1':
(BytesIO(b'testfile2'), 'test2.jpg')
})
a1 = Attachment.get(Attachment.filename == 'test1.txt')
a2 = Attachment.get(Attachment.filename == 'test2.jpg')
a1_data = {
'data': '/documents/1/attachments/test1.txt/download/',
'data_length': 9,
'mimetype': 'text/plain',
'timestamp': str(a1.timestamp),
'filename': 'test1.txt'
}
a2_data = {
'data': '/documents/1/attachments/test2.jpg/download/',
'data_length': 9,
'mimetype': 'image/jpeg',
'timestamp': str(a2.timestamp),
'filename': 'test2.jpg'
}
resp_data = json_load(response.data)
self.assertEqual(
resp_data, {
'attachments': [a1_data, a2_data],
'content': 'doc a',
'id': 1,
'identifier': None,
'indexes': ['idx-a'],
'metadata': {
'k1': 'v1-a',
'k2': 'v2-a'
}
})
Attachment.update(timestamp='2016-02-01 01:02:03').execute()
with assert_query_count(3):
resp = self.app.get('/documents/1/attachments/')
self.assertEqual(
json_load(resp.data), {
'ordering': [],
'pages':
1,
'page':
1,
'attachments': [
{
'mimetype': 'text/plain',
'timestamp': '2016-02-01 01:02:03',
'data_length': 9,
'filename': 'test1.txt',
'document': '/documents/1/',
'data': '/documents/1/attachments/test1.txt/download/',
},
{
'mimetype': 'image/jpeg',
'timestamp': '2016-02-01 01:02:03',
'data_length': 9,
'filename': 'test2.jpg',
'document': '/documents/1/',
'data': '/documents/1/attachments/test2.jpg/download/',
},
],
})
def test_blob_caching(method, endpoint, client, app):
digest = "sha256:" + hashlib.sha256(b"a").hexdigest()
location = ImageStorageLocation.get(name="local_us")
model.blob.store_blob_record_and_temp_link("devtable", "simple", digest,
location, 1, 10000000)
params = {
"repository": "devtable/simple",
"digest": digest,
}
user = model.user.get_user("devtable")
access = [{
"type": "repository",
"name": "devtable/simple",
"actions": ["pull"],
}]
context, subject = build_context_and_subject(
ValidatedAuthContext(user=user))
token = generate_bearer_token(realapp.config["SERVER_HOSTNAME"], subject,
context, access, 600, instance_keys)
headers = {
"Authorization": "Bearer %s" % token.decode("ascii"),
}
# Run without caching to make sure the request works. This also preloads some of
# our global model caches.
conduct_call(client,
"v2." + endpoint,
url_for,
method,
params,
expected_code=200,
headers=headers)
# First request should make a DB query to retrieve the blob.
conduct_call(client,
"v2." + endpoint,
url_for,
method,
params,
expected_code=200,
headers=headers)
# turn off pull-thru proxy cache. it adds an extra query to the pull operation
# for checking whether the namespace is a cache org or not before retrieving
# the blob.
with patch("endpoints.decorators.features.PROXY_CACHE", False):
# Subsequent requests should use the cached blob.
with assert_query_count(0):
conduct_call(
client,
"v2." + endpoint,
url_for,
method,
params,
expected_code=200,
headers=headers,
)