def test_as_completed(self): tasks = [FutureTask("group_id", str(n), client=self.client) for n in range(5)] response1 = [{'id': str(n), 'result_type': 'json'} for n in range(3)] response2 = [{'id': str(n), 'result_type': 'json'} for n in range(3, 5)] self.mock_response(responses.POST, {'results': response1}) self.mock_response(responses.POST, {'results': response2}) completed_tasks = list(as_completed(tasks, show_progress=False)) self.assertEqual(5, len(completed_tasks)) self.assertEqual(list(range(5)), [int(r._task_result['id']) for r in completed_tasks])
def test_as_completed(self): tasks = [ FutureTask("group_id", str(n), client=self.client) for n in range(5) ] response1 = [{"id": str(n), "result_type": "json"} for n in range(3)] response2 = [{ "id": str(n), "result_type": "json" } for n in range(3, 5)] self.mock_response( responses.GET, { "id": "foo", "queue": { "pending": 3, "successes": 0, "failures": 0 }, "status": "running", }, ) self.mock_response( responses.GET, { "id": "foo", "queue": { "pending": 3, "successes": 0, "failures": 0 }, "status": "running", }, ) self.mock_response(responses.POST, {"results": response1}) self.mock_response(responses.POST, {"results": response2}) completed_tasks = list(as_completed(tasks, show_progress=False)) assert 5 == len(completed_tasks) assert list( range(5)) == [int(r._task_result["id"]) for r in completed_tasks]
def test_as_completed_exception(self): tasks = [ FutureTask("group_id", str(n), client=self.client) for n in range(5) ] response1 = [{"id": str(n), "result_type": "json"} for n in range(3)] response2 = [{ "id": str(n), "result_type": "json" } for n in range(3, 5)] self.mock_response( responses.GET, { "id": "foo", "queue": { "pending": 3, "successes": 0, "failures": 0 }, "status": "running", }, ) self.mock_response( responses.GET, { "id": "foo", "queue": { "pending": 3, "successes": 0, "failures": 0 }, "status": "terminated", }, ) self.mock_response(responses.POST, {"results": response1}) self.mock_response(responses.POST, {"results": response2}) with pytest.raises(GroupTerminalException): list(as_completed(tasks, show_progress=False))
def _run_platform_tasks(self, async_func): self.tasks = async_func.map(self.args_list) nb_tasks = len(self.tasks) sample_ids = [t.tuid for t in self.tasks[:4]] if nb_tasks > 4: elps = ['...'] else: elps = [] self._print("group_id: {}".format(self.tasks[0].guid)) self._print("nb_tasks: {}".format(nb_tasks)) self._print("task_ids: {}".format(sample_ids + elps)) self._response_divider(True, False) for task in as_completed(self.tasks): if self.noisy: utils.vspace(1) if task.is_success: self._print_result(task.result) else: utils.line("*") self._print(task.exception, plain_text=True) self._print(task.log, plain_text=True) utils.line("*")
def test_as_completed_exception(self): tasks = [ FutureTask("group_id", str(n), client=self.client) for n in range(5) ] response1 = [{'id': str(n), 'result_type': 'json'} for n in range(3)] response2 = [{ 'id': str(n), 'result_type': 'json' } for n in range(3, 5)] self.mock_response( responses.GET, { 'id': 'foo', 'queue': { 'pending': 3, 'successes': 0, 'failures': 0 }, 'status': 'running' }) self.mock_response( responses.GET, { 'id': 'foo', 'queue': { 'pending': 3, 'successes': 0, 'failures': 0 }, 'status': 'terminated' }) self.mock_response(responses.POST, {'results': response1}) self.mock_response(responses.POST, {'results': response2}) with self.assertRaises(GroupTerminalException): list(as_completed(tasks, show_progress=False))
from descarteslabs.client.services.tasks import Tasks, as_completed at = Tasks() def f(x, option=None): return "Hello World {} {}".format(x, option) async_func = at.create_function( f, name='hello-world', image="us.gcr.io/dl-ci-cd/images/tasks/public/py2/default:v2018.07.25", ) task1 = async_func(1) task2 = async_func(2, option='hi') print(task1.result) print(task2.result) tasks = async_func.map(range(100)) for task in as_completed(tasks): print(task.result)
def deploy(model_version, output_product): deploy_aoi = { "type": "Polygon", "coordinates": [[ [-99.24164417538321, 26.138411465362807], [-93.37666136803256, 26.138411465362807], [-93.37666136803256, 31.060649553995205], [-99.24164417538321, 31.060649553995205], [-99.24164417538321, 26.138411465362807], ]], } # Make sure the output product exists try: dl.Catalog().get_product(output_product) except dl.client.exceptions.NotFoundError: print("Output product {} does not exist".format(output_product)) return # Decompose our AOI into 1024x1024 pixel tiles at 90m resolution in UTM tiles = dl.scenes.DLTile.from_shape(deploy_aoi, resolution=90.0, tilesize=1024, pad=0) # Register our prediction function in the Tasks environment. # # We specify the resource requirements per worker (1 CPU & 2GB of RAM), # the environment (container with Python 3.7), and any extra PyPI # requirements (descarteslabs client and scikit-learn). tasks = dl.Tasks() run_model_remotely = tasks.create_function( run_model, name="example water model deployment", image= "us.gcr.io/dl-ci-cd/images/tasks/public/py3.7/default:v2019.05.29", cpu=1.0, memory="2Gi", requirements=[ "descarteslabs[complete]==0.19.0", "scikit-learn==0.21.1" ], ) # Create a list with arguments of each invocation of run_model task_arguments = [(model_version, dltile.key, output_product) for dltile in tiles] results = run_model_remotely.map(*zip(*task_arguments)) print("Submitted {} tasks to task group {}...".format( len(tiles), run_model_remotely.group_id)) # Iterate through task results as they complete. # # If some of the tasks failed, we will print the console output and the # arguments of that invocation. # # Note that this is for informational purposes only, and the tasks will # continue running if the script is interrupted at this point. You can use # https://monitor.descarteslabs.com/ to see the status of all running # task groups. for i, task in enumerate(as_completed(results, show_progress=False)): percent_complete = 100.0 * i / len(results) print( "Progress update: {} completed out of {} ({:.2f}%) - last task took {:.2f}sec to {}" .format( i + 1, len(results), percent_complete, task.runtime, "succeed" if task.is_success else "fail", )) if not task.is_success: print("\nTASK FAILURE with arguments {}:\n{}".format( task.args, task.log.decode())) # Clean up the task group tasks.delete_group_by_id(run_model_remotely.group_id)
from descarteslabs.client.services.tasks import Tasks, as_completed def f(): import tensorflow as tf hello = tf.constant('Hello, TensorFlow!') # Start tf session sess = tf.Session() # Run the op return sess.run(hello) at = Tasks() async_function = at.create_function( f, name="hello-tensorflow", image="us.gcr.io/dl-ci-cd/images/tasks/public/py2/default:v2018.07.25", ) for task in as_completed([async_function()]): print(task.result)