async def test_add_sync_utility(guillotina, loop):
requester = await guillotina
app = getUtility(IApplication, name='root')
app.add_async_utility(QUEUE_UTILITY_CONFIG, loop)
util = getUtility(IQueueUtility)
var = []
async def printHi(msg):
asyncio.sleep(0.01)
var.append(msg)
request = utils.get_mocked_request(requester.db)
root = await utils.get_root(request)
await util.add(AsyncMockView(root, request, printHi, 'hola1'))
await util.add(AsyncMockView(root, request, printHi, 'hola2'))
await util.add(AsyncMockView(root, request, printHi, 'hola3'))
await util.add(AsyncMockView(root, request, printHi, 'hola4'))
await util._queue.join()
assert 'hola1' in var
assert 'hola2' in var
assert 'hola3' in var
assert 'hola4' in var
assert len(var) == 4
app.del_async_utility(QUEUE_UTILITY_CONFIG)
async def load_videos(self, prefix='Movies/'):
db = await self._db.get()
if 'videos' not in db:
db['videos'] = {}
util = getUtility(IGCloudBlobStore)
async with aiohttp.ClientSession() as session:
access_token = await util.get_access_token()
url = '{}/vangheem-media/o'.format(OBJECT_BASE_URL)
resp = await session.get(
url,
headers={'AUTHORIZATION': 'Bearer %s' % access_token},
params={'prefix': prefix})
data = await resp.json()
for video in data['items']:
filename = video['name']
if filename == prefix:
continue
ext = filename.split('.')[-1].lower()
if ext not in ('m4v', 'mov', 'mp4'):
continue
video = {
'name': filename,
'id': video['id'],
'created': video['timeCreated'],
'updated': video['updated'],
'link': video['mediaLink'],
'size': video['size'],
'selfLink': video['selfLink']
}
db['videos'][video['id']] = video
await self._queue.put(video)
await self._db.save()
async def es_migrate(path,
root,
request,
reindex_security=False,
mapping_only=False,
full=False,
force=False):
try:
ob, end_path = await traverse(request, root,
path.lstrip('/').split('/'))
if len(end_path) != 0:
raise Exception('Could not found object')
search = getUtility(ICatalogUtility)
migrator = Migrator(search,
ob,
reindex_security=reindex_security,
full=full,
force=force,
mapping_only=mapping_only,
request=request,
log_details=True)
await migrator.run_migration()
finally:
txn = get_transaction(request)
if txn is not None:
tm = get_tm(request)
await tm.abort(txn=txn)
async def migrate_all(self, arguments):
search = getUtility(ICatalogUtility)
change_transaction_strategy('none')
await asyncio.sleep(1) # since something initialize custom types...
async for _, tm, container in get_containers(self.request):
try:
self.migrator = Migrator(
search,
container,
response=printer(),
full=arguments.full,
force=arguments.force,
log_details=arguments.log_details,
memory_tracking=arguments.memory_tracking,
reindex_security=arguments.reindex_security,
mapping_only=arguments.mapping_only)
await self.migrator.run_migration()
seconds = int(time.time() - self.migrator.start_time)
logger.warning(f'''Finished migration:
Total Seconds: {seconds}
Processed: {self.migrator.processed}
Indexed: {self.migrator.indexed}
Objects missing: {len(self.migrator.missing)}
Objects orphaned: {len(self.migrator.orphaned)}
Mapping Diff: {self.migrator.mapping_diff}
''')
finally:
await tm.commit(self.request)
async def _test_new_deletes_are_performed_during_migration(es_requester):
async with es_requester as requester:
await add_content(requester)
container, request, txn, tm = await setup_txn_on_container(requester)
search = getUtility(ICatalogUtility)
migrator = Migrator(search, container, force=True, request=request)
await migrator.setup_next_index()
await migrator.copy_to_next_index()
await search.refresh(container, migrator.work_index_name)
await search.refresh(container)
num_docs = await search.get_doc_count(container, migrator.work_index_name)
current_docs = await search.get_doc_count(container)
assert num_docs == current_docs
keys = await container.async_keys()
key = random.choice(keys)
ob = await container.async_get(key)
keys = await ob.async_keys()
key = random.choice(keys)
ob = await ob.async_get(key)
await search.remove(container, [(
ob._p_oid, ob.type_name, get_content_path(ob)
)], request=request)
await search.refresh(container, migrator.work_index_name)
await search.refresh(container)
num_docs = await search.get_doc_count(container, migrator.work_index_name)
current_count = await search.get_doc_count(container)
assert num_docs == current_count
async def _test_new_indexes_are_performed_during_migration(es_requester):
async with es_requester as requester:
await add_content(requester)
container, request, txn, tm = await setup_txn_on_container(requester)
search = getUtility(ICatalogUtility)
migrator = Migrator(search, container, force=True, request=request)
await migrator.setup_next_index()
await migrator.copy_to_next_index()
await asyncio.sleep(1)
await search.refresh(container, migrator.work_index_name)
await search.refresh(container)
await asyncio.sleep(1)
num_docs = await search.get_doc_count(container, migrator.work_index_name)
assert num_docs == await search.get_doc_count(container)
await add_content(requester, base_id='foobar1-')
await asyncio.sleep(1)
await search.refresh(container, migrator.work_index_name)
await search.refresh(container)
await asyncio.sleep(1)
num_docs = await search.get_doc_count(container, migrator.work_index_name)
assert num_docs == await search.get_doc_count(container)
async def test_removes_orphans(es_requester):
async with es_requester as requester:
container, request, txn, tm = await setup_txn_on_container(requester)
search = getUtility(ICatalogUtility)
await search.index(
container, {'foobar': {
'title': 'foobar',
'type_name': 'Item'
}})
# foobar here is an orphaned object because it doesn't reference an object
index_name = await search.get_index_name(container) # alias
doc = await search.conn.get(index=index_name,
doc_type=DOC_TYPE,
id='foobar')
assert doc['found']
migrator = Migrator(search, container, force=True)
await migrator.run_migration()
async def _test():
with pytest.raises(aioelasticsearch.exceptions.NotFoundError):
await search.conn.get(index=index_name,
doc_type=DOC_TYPE,
id='foobar')
assert len(migrator.orphaned) == 1
assert migrator.orphaned[0] == 'foobar'
await run_with_retries(_test, requester)
async def test_calculate_mapping_diff(es_requester):
async with es_requester as requester:
container, request, txn, tm = await setup_txn_on_container(requester)
search = getUtility(ICatalogUtility)
migrator = Migrator(search, container, force=True, request=request)
version, new_index_name = await migrator.create_next_index()
migrator.work_index_name = new_index_name
mappings = get_mappings()
index_settings = DEFAULT_SETTINGS.copy()
index_settings.update(app_settings.get('index', {}))
# tweak mappings so we can get the diff...
if 'creators' in mappings['properties']:
mappings['properties']['creators']['type'] = 'text'
mappings['properties']['foobar'] = {'type': 'keyword', 'index': True}
await search.conn.indices.close(new_index_name)
await search.conn.indices.put_settings(body=index_settings,
index=new_index_name)
await search.conn.indices.put_mapping(index=new_index_name,
doc_type=DOC_TYPE,
body=mappings)
await search.conn.indices.open(new_index_name)
diff = await migrator.calculate_mapping_diff()
assert len(diff[DOC_TYPE]) == 2
async def test_allowed_types(self, dummy_request):
self.request = dummy_request
utils.login(self.request)
site = await create_content('Site',
id='guillotina',
title='Guillotina')
site.__name__ = 'guillotina'
utils._p_register(site)
import guillotina.tests
configure.register_configuration(
Folder,
dict(
portal_type="TestType",
allowed_types=['Item'],
module=guillotina.tests # for registration initialization
),
'contenttype')
root = getUtility(IApplication, name='root')
configure.load_configuration(root.app.config, 'guillotina.tests',
'contenttype')
root.app.config.execute_actions()
load_cached_schema()
obj = await create_content_in_container(site, 'TestType', 'foobar')
constrains = IConstrainTypes(obj, None)
assert constrains.get_allowed_types() == ['Item']
assert constrains.is_type_allowed('Item')
with pytest.raises(NotAllowedContentType):
await create_content_in_container(obj, 'TestType', 'foobar')
await create_content_in_container(obj, 'Item', 'foobar')
def get_cached_factory(portal_type):
if portal_type in FACTORY_CACHE:
factory = FACTORY_CACHE[portal_type]
else:
factory = getUtility(IResourceFactory, portal_type)
FACTORY_CACHE[portal_type] = factory
return factory
async def test_calculate_mapping_diff(es_requester):
async with es_requester as requester:
container, request, txn, tm = await setup_txn_on_container(requester)
search = getUtility(ICatalogUtility)
migrator = Migrator(search, container, force=True, request=request)
version, new_index_name = await migrator.create_next_index()
migrator.work_index_name = new_index_name
mappings = get_mappings()
index_settings = DEFAULT_SETTINGS.copy()
index_settings.update(app_settings.get('index', {}))
# tweak mappings so we can get the diff...
for key, value in mappings.items():
# need to modify on *all* or it won't work with ES..
if 'creators' in value['properties']:
value['properties']['creators']['type'] = 'text'
mappings['Item']['properties']['foobar'] = {
'type': 'keyword',
'index': True
}
await search.conn.indices.close(new_index_name)
await search.conn.indices.put_settings(index_settings, new_index_name)
for key, value in mappings.items():
await search.conn.indices.put_mapping(new_index_name, key, value)
await search.conn.indices.open(new_index_name)
diff = await migrator.calculate_mapping_diff()
assert len(diff['Folder']) == 1
assert len(diff['Item']) == 2
def test_negotiate_complex_accept_header(dummy_guillotina):
np = getUtility(IContentNegotiation, 'content_type')
ap = np.negotiate(accept='application/vnd.google.protobuf;'
'proto=io.prometheus.client.MetricFamily;'
'encoding=delimited;q=0.7,text/plain;'
'version=0.0.4;q=0.3,*/*;q=0.1')
assert str(ap.content_type) == 'text/plain'
def get_cached_factory(type_name):
if type_name in FACTORY_CACHE:
factory = FACTORY_CACHE[type_name]
else:
factory = getUtility(IResourceFactory, type_name)
FACTORY_CACHE[type_name] = factory
return factory
def del_async_utility(self, config):
self.cancel_async_utility(config['provides'])
interface = import_class(config['provides'])
utility = getUtility(interface)
gsm = getGlobalSiteManager()
gsm.unregisterUtility(utility, provided=interface)
del self._async_utilities[config['provides']]
def __init__(self,
txn,
tm,
request,
container,
last_tid=-2,
index_scroll='15m',
hits_scroll='5m',
use_tid_query=True):
self.txn = txn
self.tm = tm
self.request = request
self.container = container
self.orphaned = set()
self.missing = set()
self.out_of_date = set()
self.utility = getUtility(ICatalogUtility)
self.migrator = Migrator(self.utility,
self.container,
full=True,
bulk_size=10)
self.cache = LRU(200)
self.last_tid = last_tid
print(f'Last TID: {self.last_tid}')
self.use_tid_query = use_tid_query
self.last_zoid = None
# for state tracking so we get boundries right
self.last_result_set = []
self.index_scroll = index_scroll
self.hits_scroll = hits_scroll
async def test_removes_orphans(es_requester):
async with es_requester as requester:
container, request, txn, tm = await setup_txn_on_container(requester)
search = getUtility(ICatalogUtility)
await search.index(
container, {'foobar': {
'title': 'foobar',
'type_name': 'Item'
}})
# foobar here is an orphaned object because it doesn't reference an object
index_name = await search.get_index_name(container) # alias
doc = await search.conn.get(index_name, 'foobar')
assert doc['found']
migrator = Migrator(search, container, force=True)
await migrator.run_migration()
await asyncio.sleep(1)
await search.refresh(container, index_name)
await asyncio.sleep(1)
with pytest.raises(aioes.exception.NotFoundError):
doc = await search.conn.get(index_name, 'foobar')
assert len(migrator.orphaned) == 1
assert migrator.orphaned[0] == 'foobar'
async def test_subscriber_ignores_trsn_on_invalidate(
redis_container, dummy_guillotina, loop):
await cache.close_redis_pool()
trns = mocks.MockTransaction(mocks.MockTransactionManager())
trns.added = trns.deleted = {}
content = create_content()
trns.modified = {content._p_oid: content}
rcache = RedisCache(trns, loop=loop)
await rcache.clear()
await rcache.set('foobar', oid=content._p_oid)
assert serialize.loads(
await rcache._redis.get(
CACHE_PREFIX + 'root-' + content._p_oid)) == "foobar"
assert rcache._memory_cache.get('root-' + content._p_oid) == 'foobar'
assert await rcache.get(oid=content._p_oid) == 'foobar'
assert 'root-' + content._p_oid in rcache._memory_cache
utility = getUtility(IRedisChannelUtility)
utility.ignore_tid(5555)
await rcache._redis.publish(
app_settings['redis']['updates_channel'], serialize.dumps({
'tid': 5555,
'keys': ['root-' + content._p_oid]
}))
await asyncio.sleep(1) # should be enough for pub/sub to finish
# should still be there because we set to ignore this tid
assert 'root-' + content._p_oid in rcache._memory_cache
# tid should also now be removed from ignored list
assert 5555 not in utility._ignored_tids
await cache.close_redis_pool()
async def _test_migrate_while_content_getting_added(es_requester):
async with es_requester as requester:
add_count = await add_content(requester)
container, request, txn, tm = await setup_txn_on_container(requester)
search = getUtility(ICatalogUtility)
await search.refresh(container)
await asyncio.sleep(3)
assert add_count == await search.get_doc_count(container)
migrator = Migrator(search, container, force=True)
add_content_task1 = asyncio.ensure_future(
add_content(requester, base_id='foo1-'))
add_content_task2 = asyncio.ensure_future(
add_content(requester, base_id='foo2-'))
reindex_task = asyncio.ensure_future(migrator.run_migration())
await asyncio.wait(
[add_content_task1, reindex_task, add_content_task2])
await search.refresh(container)
await asyncio.sleep(3)
idx_count = await search.get_doc_count(container)
# +1 here because container ob now indexed and it isn't by default in tests
assert (add_count * 3) + 1 == idx_count
await tm.abort(txn=txn)
async def test_migrator_emmits_events_on_end(es_requester, event_handler):
async with es_requester as requester:
resp, status = await requester('POST',
'/db/guillotina/',
data=json.dumps({
'@type': 'Folder',
'title': 'Folder',
'id': 'foobar'
}))
container, req, txn, tm = await setup_txn_on_container(requester)
search = getUtility(ICatalogUtility)
gr.base.adapters.subscribe([IIndexProgress], None,
event_handler.subscribe)
migrator = Reindexer(search,
container,
force=True,
request=req,
reindex_security=True)
ob = await container.async_get('foobar')
await migrator.reindex(ob)
assert event_handler.called == True
assert len(event_handler.event) == 2
assert event_handler.event[0].completed == None
assert event_handler.event[0].processed == 0
assert event_handler.event[1].completed == True
assert event_handler.event[0].context == container
async def test_write_large_blob_data(postgres, guillotina_main):
root = getUtility(IApplication, name='root')
db = root['db']
request = get_mocked_request(db)
login(request)
async with managed_transaction(request=request):
container = await create_content_in_container(
db, 'Container', 'container', request=request,
title='Container')
blob = Blob(container)
container.blob = blob
multiplier = 999999
blobfi = blob.open('w')
await blobfi.async_write(b'foobar' * multiplier)
async with managed_transaction(request=request):
container = await db.async_get('container')
assert await container.blob.open().async_read() == (b'foobar' * multiplier)
assert container.blob.size == len(b'foobar' * multiplier)
assert container.blob.chunks == 6
await db.async_del('container')
async def get_video(video_id):
util = getUtility(IGlexUtility)
db = await util.get_db()
try:
return db['videos'][video_id]
except IndexError:
return None
async def download_task(video):
util = getUtility(IGCloudBlobStore)
if not os.path.exists(app_settings['download_folder']):
os.mkdir(app_settings['download_folder'])
filepath = os.path.join(app_settings['download_folder'],
_get_filename(video))
fi = open(filepath, 'wb')
os.utime(filepath, None) # force filename...
print(f'Downloading {video["id"]}')
async with aiohttp.ClientSession() as session:
api_resp = await session.get(video['link'],
headers={
'AUTHORIZATION':
'Bearer %s' %
await util.get_access_token()
})
count = 0
# file_size = int(video['size'])
while True:
chunk = await api_resp.content.read(1024 * 1024 * 5)
if len(chunk) > 0:
count += len(chunk)
# print("Download {}%.".format(int((count / file_size) * 100)))
fi.write(chunk)
fi.flush()
os.fsync(fi.fileno())
else:
break
fi.close()
print(f'Finished download {video["id"]}')
if video['id'] in _tasks:
del _tasks[video['id']]
async def handle_ws_request(self, ws, message):
method = app_settings['http_methods']['GET']
path = tuple(p for p in message['value'].split('/') if p)
# avoid circular import
from guillotina.traversal import do_traverse
obj, tail = await do_traverse(self.request, self.request.container,
path)
traverse_to = None
if tail and len(tail) == 1:
view_name = tail[0]
elif tail is None or len(tail) == 0:
view_name = ''
else:
view_name = tail[0]
traverse_to = tail[1:]
permission = getUtility(IPermission, name='guillotina.AccessContent')
allowed = IInteraction(self.request).check_permission(
permission.id, obj)
if not allowed:
response = {'error': 'Not allowed'}
ws.send_str(ujson.dumps(response))
try:
view = queryMultiAdapter((obj, self.request),
method,
name=view_name)
except AttributeError:
view = None
if traverse_to is not None:
if view is None or not ITraversableView.providedBy(view):
response = {'error': 'Not found'}
ws.send_str(ujson.dumps(response))
else:
try:
view = await view.publish_traverse(traverse_to)
except Exception as e:
logger.error("Exception on view execution", exc_info=e)
response = {'error': 'Not found'}
ws.send_str(ujson.dumps(response))
view_result = await view()
if isinstance(view_result, Response):
view_result = view_result.response
# Return the value
ws.send_str(ujson.dumps(view_result))
# Wait for possible value
futures_to_wait = self.request._futures.values()
if futures_to_wait:
await asyncio.gather(*list(futures_to_wait))
self.request._futures = {}
async def download_head(context, request):
util = getUtility(IGlexUtility)
db = await util.get_db()
try:
video = db['videos'][request.GET['id']]
except IndexError:
return HTTPNotFound()
return Response(headers={'Content-Length': video['size']})
async def test_create_next_index(es_requester):
async with es_requester as requester:
container, request, txn, tm = await setup_txn_on_container(requester)
search = getUtility(ICatalogUtility)
migrator = Migrator(search, container, force=True, request=request)
version, name = await migrator.create_next_index()
assert version == 2
assert name == 'guillotina-guillotina_2'
def dummy_request(dummy_guillotina, monkeypatch):
from guillotina.interfaces import IApplication
from guillotina.component import getUtility
root = getUtility(IApplication, name='root')
db = root['db']
request = get_mocked_request(db)
return request
async def test_register_behavior(container_requester):
cur_count = len(
configure.get_configurations('guillotina.tests', 'behavior'))
from guillotina.interfaces import IFormFieldProvider, IResource
from zope.interface import provider
from guillotina import schema
@provider(IFormFieldProvider)
class IMyBehavior(Interface):
foobar = schema.Text()
class IMyBehavior2(Interface):
foobar = schema.Text()
configure.behavior(
title="MyBehavior",
provides=IMyBehavior,
factory="guillotina.behaviors.instance.AnnotationBehavior",
for_="guillotina.interfaces.IResource"
)()
configure.behavior(
title="MyBehavior2",
provides=IMyBehavior2,
factory="guillotina.behaviors.instance.AnnotationBehavior",
for_="guillotina.interfaces.IResource"
)()
assert len(configure.get_configurations('guillotina.tests', 'behavior')) == cur_count + 2
class IMyType(IResource):
pass
class MyType(Item):
pass
configure.register_configuration(MyType, dict(
context=IContainer,
schema=IMyType,
type_name="MyType2",
behaviors=[IMyBehavior]
), 'contenttype')
root = getUtility(IApplication, name='root')
config = root.app.config
# now test it...
configure.load_configuration(config, 'guillotina.tests', 'contenttype')
configure.load_configuration(config, 'guillotina.tests', 'behavior')
config.execute_actions()
async with await container_requester as requester:
response, status = await requester('GET', '/db/guillotina/@types')
type_ = [s for s in response if s['title'] == 'MyType2'][0]
assert 'foobar' in type_['definitions']['IMyBehavior']['properties']
# also get_all_possible_schemas_for_type should come with this new behavior
behaviors_schemas = get_all_possible_schemas_for_type('MyType2')
assert IMyBehavior2 in behaviors_schemas
async def get_dbs(self):
root = getUtility(IApplication, name='root')
for _id, db in root:
if IDatabase.providedBy(db):
tm = db.get_transaction_manager()
tm.request = self.request
await tm.begin(self.request)
async for s_id, container in db.async_items():
tm.request.container = container
yield tm, container
async def get_containers(request):
root = getUtility(IApplication, name='root')
for _id, db in root:
if IDatabase.providedBy(db):
db._db._storage._transaction_strategy = 'none'
tm = db.get_transaction_manager()
tm.request = request
txn = await tm.begin(request)
async for s_id, container in db.async_items():
tm.request.container = container
yield txn, tm, container
async def test_updates_index_name(es_requester):
async with es_requester as requester:
container, request, txn, tm = await setup_txn_on_container(requester)
search = getUtility(ICatalogUtility)
existing_index = await search.get_real_index_name(container)
assert await search.conn.indices.exists(existing_index)
migrator = Migrator(search, container, force=True, request=request)
await migrator.run_migration()
assert not await search.conn.indices.exists(existing_index)
assert search.conn.indices.exists(migrator.work_index_name)
assert await search.get_real_index_name(container
) == migrator.work_index_name
