def send_pipeline_and_wait_finished(): pipeline_manager = PipelineManager(api=API_ADDRESS, broadcast=BROADCAST_ADDRESS) pipelines = [] for i in range(10): pipeline = Pipeline( { Job(u'worker_1'): Job(u'worker_2'), Job(u'worker_2'): Job(u'worker_3') }, data={'index': i}) pipeline_manager.start(pipeline) pipelines.append(pipeline) assert pipeline_manager.started_pipelines == 10 assert pipeline_manager.finished_pipelines == 0 start = time() pipeline_manager.finished(pipelines[0]) # only for testing this method while pipeline_manager.finished_pipelines < pipeline_manager.started_pipelines: pipeline_manager.update(0.5) end = time() pipeline_manager.disconnect() return { 'duration': pipeline.duration, 'real_duration': end - start, 'finished_pipelines': pipeline_manager.finished_pipelines, 'started_pipelines': pipeline_manager.started_pipelines }
def verify_PipelineManager_exceptions(): pipeline_1 = Pipeline({ Job(u'worker_1'): Job(u'worker_2'), Job(u'worker_2'): Job(u'worker_3') }) pipeline_2 = Pipeline({Job(u'worker_1'): Job(u'worker_2')}) pipeline_manager = PipelineManager(api=API_ADDRESS, broadcast=BROADCAST_ADDRESS) pipeline_manager.start(pipeline_1) raise_1, raise_2 = False, False try: pipeline_manager.start(pipeline_1) except ValueError: raise_1 = True try: pipeline_manager.finished(pipeline_2) except ValueError: raise_2 = True pipeline_manager.disconnect() return { 'raise_1': raise_1, 'raise_2': raise_2, 'started_at': pipeline_1.started_at }
def send_pipeline(): pipeline = Pipeline({Job(u'worker_1'): Job(u'worker_2'), Job(u'worker_2'): Job(u'worker_3')}) pipeline_manager = PipelineManager(api=API_ADDRESS, broadcast=BROADCAST_ADDRESS) before = pipeline.id pipeline_id = pipeline_manager.start(pipeline) pipeline_manager.disconnect() return before, pipeline_id, pipeline.id
def send_pipeline(): pipeline = Pipeline({Job(u'worker_1'): Job(u'worker_2'), Job(u'worker_2'): Job(u'worker_3')}) pipeline_manager = PipelineManager(api='tcp://localhost:5550', broadcast='tcp://localhost:5551') before = pipeline.id pipeline_id = pipeline_manager.start(pipeline) pipeline_manager.disconnect() return before, pipeline_id, pipeline.id
def send_pipeline(): pipeline = Pipeline({ Job(u'worker_1'): Job(u'worker_2'), Job(u'worker_2'): Job(u'worker_3') }) pipeline_manager = PipelineManager(api=API_ADDRESS, broadcast=BROADCAST_ADDRESS) before = pipeline.id pipeline_id = pipeline_manager.start(pipeline) pipeline_manager.disconnect() return before, pipeline_id, pipeline.id
def send_pipeline_and_wait_finished(): import time pipeline = Pipeline({Job(u'worker_1'): Job(u'worker_2'), Job(u'worker_2'): Job(u'worker_3')}) pipeline_manager = PipelineManager(api='tcp://localhost:5550', broadcast='tcp://localhost:5551') pipeline_manager.start(pipeline) start = time.time() while not pipeline_manager.finished(pipeline): time.sleep(0.1) end = time.time() pipeline_manager.disconnect() return {'duration': pipeline.duration, 'real_duration': end - start}
def verify_PipelineManager_exceptions(): pipeline_1 = Pipeline({Job(u'worker_1'): Job(u'worker_2'), Job(u'worker_2'): Job(u'worker_3')}) pipeline_2 = Pipeline({Job(u'worker_1'): Job(u'worker_2')}) pipeline_manager = PipelineManager(api=API_ADDRESS, broadcast=BROADCAST_ADDRESS) pipeline_manager.start(pipeline_1) raise_1, raise_2 = False, False try: pipeline_manager.start(pipeline_1) except ValueError: raise_1 = True try: pipeline_manager.finished(pipeline_2) except ValueError: raise_2 = True pipeline_manager.disconnect() return {'raise_1': raise_1, 'raise_2': raise_2, 'started_at': pipeline_1.started_at}
def verify_PipelineManager_exceptions(): pipeline_1 = Pipeline({Job(u'worker_1'): Job(u'worker_2'), Job(u'worker_2'): Job(u'worker_3')}) pipeline_2 = Pipeline({Job(u'worker_1'): Job(u'worker_2')}) pipeline_manager = PipelineManager(api='tcp://localhost:5550', broadcast='tcp://localhost:5551') pipeline_manager.start(pipeline_1) raise_1, raise_2 = False, False try: pipeline_manager.start(pipeline_1) except ValueError: raise_1 = True try: pipeline_manager.finished(pipeline_2) except ValueError: raise_2 = True pipeline_manager.disconnect() return {'raise_1': raise_1, 'raise_2': raise_2, 'started_at': pipeline_1.started_at}
def send_pipeline_and_wait_finished(): pipeline_manager = PipelineManager(api=API_ADDRESS, broadcast=BROADCAST_ADDRESS) pipelines = [] for i in range(10): pipeline = Pipeline({Job(u'worker_1'): Job(u'worker_2'), Job(u'worker_2'): Job(u'worker_3')}, data={'index': i}) pipeline_manager.start(pipeline) pipelines.append(pipeline) assert pipeline_manager.started_pipelines == 10 assert pipeline_manager.finished_pipelines == 0 start = time() pipeline_manager.finished(pipelines[0]) # only for testing this method while pipeline_manager.finished_pipelines < pipeline_manager.started_pipelines: pipeline_manager.update(0.5) end = time() pipeline_manager.disconnect() return {'duration': pipeline.duration, 'real_duration': end - start, 'finished_pipelines': pipeline_manager.finished_pipelines, 'started_pipelines': pipeline_manager.started_pipelines}