async def migrate_all(self, arguments):
search = get_utility(ICatalogUtility)
change_transaction_strategy('none')
await asyncio.sleep(1) # since something initialize custom types...
async for _, tm, container in get_containers(self.request):
try:
self.migrator = Migrator(
search,
container,
response=printer(),
full=arguments.full,
force=arguments.force,
log_details=arguments.log_details,
memory_tracking=arguments.memory_tracking,
reindex_security=arguments.reindex_security,
mapping_only=arguments.mapping_only,
cache=False)
await self.migrator.run_migration()
seconds = int(time.time() - self.migrator.start_time)
logger.warning(f'''Finished migration:
Total Seconds: {seconds}
Processed: {self.migrator.processed}
Indexed: {self.migrator.indexed}
Objects missing: {len(self.migrator.missing)}
Objects orphaned: {len(self.migrator.orphaned)}
Mapping Diff: {self.migrator.mapping_diff}
''')
finally:
await tm.commit(self.request)
async def reindex_all(self, arguments):
search = get_utility(ICatalogUtility)
await asyncio.sleep(1) # since something initialize custom types...
async for _, tm, container in get_containers():
try:
self.reindexer = Reindexer(
search,
container,
response=printer(),
log_details=arguments.log_details,
memory_tracking=arguments.memory_tracking,
reindex_security=arguments.reindex_security,
mapping_only=arguments.mapping_only,
cache=False,
)
await self.reindexer.reindex(container)
seconds = int(time.time() - self.reindexer.start_time)
logger.warning(f"""Finished reindex:
Total Seconds: {seconds}
Processed: {self.reindexer.processed}
Indexed: {self.reindexer.indexed}
Objects missing: {len(self.reindexer.missing)}
Objects orphaned: {len(self.reindexer.orphaned)}
""")
finally:
await tm.commit()
async def do_check(self, arguments, check_name):
aiotask_context.set('request', self.request)
first_run = True
while arguments.continuous or first_run:
if not first_run:
await asyncio.sleep(arguments.sleep)
else:
first_run = False
async for txn, tm, container in get_containers(self.request):
try:
kwargs = {}
if container._p_oid in self.state:
kwargs = self.state[container._p_oid]
vacuum = self.vacuum_klass(
txn, tm, self.request, container, **kwargs)
await vacuum.setup()
func = getattr(vacuum, check_name)
await func()
if vacuum.last_tid > 0:
self.state[container._p_oid] = {
'last_tid': vacuum.last_tid
}
logger.warning(f'''Finished vacuuming with results:
Orphaned cleaned: {len(vacuum.orphaned)}
Missing added: {len(vacuum.missing)}
Out of date fixed: {len(vacuum.out_of_date)}
''')
await clean_orphan_indexes(container)
except Exception:
logger.error('Error vacuuming', exc_info=True)
finally:
await tm.abort(txn=txn)
async def run(self, arguments, settings, app):
aiotask_context.set('request', self.request)
script = os.path.abspath(arguments.script)
spec = importlib.util.spec_from_file_location("module.name", script)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
if not hasattr(module, 'run'):
logger.warn(f'Not `async def run()` function found in file {script}')
return
sig = inspect.signature(module.run)
if 'container' in sig.parameters:
async for txn, tm, container in get_containers(self.request):
await module.run(container)
await tm.commit(txn=txn)
else:
await lazy_apply(module.run, app)
async def run(self, arguments, settings, app):
script = os.path.abspath(arguments.script)
spec = importlib.util.spec_from_file_location("module.name", script)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
if not hasattr(module, 'run'):
logger.warn(
f'Not `async def run()` function found in file {script}')
return
sig = inspect.signature(module.run)
if 'container' in sig.parameters:
async for txn, tm, container in get_containers():
await module.run(container)
await tm.commit(txn=txn)
else:
await lazy_apply(module.run, app)
async def test_get_containers(container_requester):
async with container_requester:
containers = [c async for c in utils.get_containers()]
assert len(containers) > 0
async def test_get_containers(container_requester):
async with container_requester as requester:
request = get_mocked_request(requester.db)
containers = [c async for c in utils.get_containers(request)]
assert len(containers) > 0
async def run(self, arguments, settings, app):
login()
utility = get_utility(IEvolutionUtility)
async for _, _, container in get_containers():
if await utility.is_installed(container):
await utility.evolve(container)
async def test_get_containers(container_requester):
async with container_requester as requester:
request = get_mocked_request(requester.db)
containers = [c async for c in utils.get_containers(request)]
assert len(containers) > 0
