def current_task(): executor = bollard.Executor() executor.start() try: async_result = executor.apply_async('current task') result = async_result.get(timeout=5) assert result is True, result finally: executor.stop()
def stop_with_running_task(): executor = bollard.Executor() executor.start() try: async_result = executor.apply_async(tasks.soft_lock) time.sleep(5) finally: executor.stop(workers=True) assert async_result.state == 'running', async_result.state
def result(): executor = bollard.Executor() executor.start() try: async_result_int = executor.apply_async(tasks.foo_int_result) async_result_str = executor.apply_async(tasks.foo_str_result) assert async_result_int.get() == 0 assert async_result_str.get() == '0' finally: executor.stop(workers=True)
def bind(): executor = bollard.Executor() executor.start() try: async_result = executor.apply_async('bind') try: async_result.get(timeout=5) except: assert False finally: executor.stop()
def add_by_name(): executor = bollard.Executor() async_result = executor.apply_async('not bar') assert isinstance(async_result, bollard.AsyncResult) task = async_result._task assert task assert task.in_db() task.load() assert task['state'] == 'pending', task['state'] assert task['name'] == 'not bar', task['name']
def callback(): executor = bollard.Executor() executor.start() try: async_result = executor.apply_async(tasks.sleep, args=(0, ), callbacks={'task.pull': _callback}) async_result.get(timeout=1) # assert async_result._task in cb.args finally: executor.stop(workers=True)
def scalarizr_process_2(): try: start_sqlite_server() executor = bollard.Executor() executor.start() try: async_result = executor.apply_async(tasks.sleep(5)) async_result.get() finally: executor.stop() except: bollard.LOG.exception(sys.exc_info())
def run(): executor = bollard.Executor() executor.start() try: async_result = executor.apply_async(tasks.foo) time.sleep(2) task = async_result._task task.load() assert task['state'] == 'completed', task['state'] assert task['start_date'] is not None assert task['end_date'] is not None finally: executor.stop(workers=True)
def args_kwds(): executor = bollard.Executor() executor.start() try: async_result = executor.apply_async(tasks.foo_args_kwds, args=(1, 2), kwds={ '1': 1, '2': 2 }) assert async_result.get() finally: executor.stop(workers=True)
def worker_restore(): max_workers = 4 executor = bollard.Executor(max_workers=max_workers) try: executor.start() time.sleep(2) pid = executor.workers[-1].pid os.kill(pid, SIGKILL) time.sleep(2) assert len(executor.workers) == max_workers for worker in executor.workers: assert worker.is_alive() finally: executor.stop(workers=True)
def get_timeout(): executor = bollard.Executor() executor.start() try: async_result = executor.apply_async(tasks.soft_lock) try: async_result.get(timeout=1) except bollard.TimeoutError: assert async_result.state == 'running', async_result.state assert async_result.result is None assert async_result.start_date is not None assert async_result.end_date is None else: assert False finally: executor.stop(workers=True)
def scalarizr_process(max_workers): try: bollard.SOFT_TIMEOUT = 10 bollard.HARD_TIMEOUT = 11 start_sqlite_server() executor = bollard.Executor(max_workers) executor.start() try: async_result1 = executor.apply_async(tasks.soft_lock) async_result2 = executor.apply_async(tasks.hard_lock) async_result1.get() async_result2.get() finally: executor.stop() except: bollard.LOG.exception(sys.exc_info())
def raise_exception(): executor = bollard.Executor(2) executor.start() try: async_result = executor.apply_async(tasks.foo_exc_result) time.sleep(5) try: async_result.get() except ZeroDivisionError: assert async_result.state == 'failed' assert async_result.traceback assert async_result.start_date is not None assert async_result.end_date is not None else: assert False finally: executor.stop(workers=True)
def revoke(): executor = bollard.Executor() executor.start() try: async_result = executor.apply_async(tasks.soft_lock) time.sleep(1) bollard.Executor.revoke(async_result.task_id) try: async_result.get() except bollard.TaskKilledError: assert async_result.state == 'failed', async_result.state assert async_result.result assert async_result.start_date is not None assert async_result.end_date is not None else: assert False finally: executor.stop(workers=True)
def hard_timeout(): executor = bollard.Executor() executor.start() try: async_result = executor.apply_async(tasks.hard_lock, soft_timeout=1, hard_timeout=5) try: async_result.get() except bollard.HardTimeLimitExceeded: assert async_result.state == 'failed', async_result.state assert async_result.result assert async_result.start_date is not None assert async_result.end_date is not None else: assert False finally: executor.stop(workers=True)
def start_stop(): executor = bollard.Executor(8) executor.start() try: time.sleep(5) assert executor.workers for w in executor.workers: assert w.is_alive() assert w._started_ev.is_set() assert w._supervisor_ev.is_set() with open(log_file) as f: text = f.read() assert 'WARNING' not in text assert 'ERROR' not in text finally: executor.stop(workers=True) time.sleep(2) for w in executor.workers: assert not w.is_alive()
def restore_after_crash(): scalarizr = multiprocessing.Process(target=scalarizr_process_2) scalarizr.start() try: while not scalarizr.is_alive(): time.sleep(1) # check what we have tasks with state 'running' conn = connect_db() conn.row_factory = sqlite3.Row conn.text_factory = sqlite3.OptimizedUnicode try: curs = conn.cursor() query = """SELECT * FROM tasks WHERE state='running'""" for i in range(10): curs.execute(query) data = curs.fetchone() if data: break time.sleep(0.1) else: assert False task = bollard.Task(**data) finally: conn.close() os.kill(scalarizr.pid, SIGKILL) time.sleep(1) assert not scalarizr.is_alive() task.load() assert task['sate'] == 'running' executor = bollard.Executor(max_workers=0) executor.start() time.sleep(10) executor.stop() task.load() assert task['state'] == 'completed', task['state'] except: bollard.LOG.exception(sys.exc_info())
def _init_bollard(self): if linux.os.windows: return # FatMouse integration import agent.celeryfile import agent.config agent.config.TASK_ENGINE = 'bollard' agent.config.HOME_DIR = \ os.path.expandvars('$ProgramData\\scalarizr') \ if linux.os.windows else \ '/var/lib/scalarizr' agent.config.CACHE_DIR = os.path.join(agent.config.HOME_DIR, 'cache') task_modules = [ 'scalarizr.api.mariadb', 'scalarizr.api.mysql', 'scalarizr.api.operation', 'scalarizr.api.postgresql', 'scalarizr.api.redis', 'scalarizr.api.storage', 'scalarizr.api.system'] + \ list(agent.celeryfile.CELERY_INCLUDE) callbacks = { 'global.push': _bollard_pass_access_data, 'global.before': _bollard_set_access_data, 'global.after': _bollard_clear_access_data, 'global.pull': _bollard_send_operation_result, 'global.fork': _bollard_fork } if not os.path.exists(bus.cnf.key_path('bollard')): bollard_key = cryptotool.keygen(length=40) bus.cnf.write_key('bollard', bollard_key, 'Bollard crypto key') bollard.CRYPTO_KEY = bus.cnf.read_key('bollard') bollard.AUTH_KEY = bollard.CRYPTO_KEY[0:16] __node__['bollard'] = bollard.Executor( task_modules=task_modules, callbacks=callbacks, push_server_address='/var/run/scalarizr.push.sock', pull_server_address='/var/run/scalarizr.pull.sock')
def soft_timeout(): executor = bollard.Executor() executor.start() try: async_result = executor.apply_async(tasks.soft_lock, soft_timeout=2, hard_timeout=10) try: async_result.get() except bollard.SoftTimeLimitExceeded: assert async_result.state == 'failed', async_result.state assert async_result.result assert async_result.start_date is not None assert async_result.end_date is not None except bollard.HardTimeLimitExceeded: if sys.platform == 'win32': assert True else: assert False else: assert False finally: executor.stop(workers=True)