def test_iterate_respects_subpolling_interval(self): r1 = self.app.AsyncResult(uuid()) r2 = self.app.AsyncResult(uuid()) backend = r1.backend = r2.backend = Mock() backend.subpolling_interval = 10 ready = r1.ready = r2.ready = Mock() def se(*args, **kwargs): ready.side_effect = KeyError() return False ready.return_value = False ready.side_effect = se x = ResultSet([r1, r2]) with self.dummy_copy(): with patch("celery.result.time") as _time: with self.assertRaises(KeyError): list(x.iterate()) _time.sleep.assert_called_with(10) backend.subpolling_interval = 0 with patch("celery.result.time") as _time: with self.assertRaises(KeyError): ready.return_value = False ready.side_effect = se list(x.iterate()) self.assertFalse(_time.sleep.called)
def test_high_priority(celery_app, celery_worker, queues): """High prio tasks should be executed first.""" # enqueue normal and high prio tasks alternately high, normal = [], [] for index in range(SAMPLES): if index % 2: normal.append( waitsome.apply_async([SLEEP], queue=queues[0], expires=TASK_EXPIRY_TIME)) else: high.append( waitsome.apply_async([SLEEP], queue=queues[0], expires=TASK_EXPIRY_TIME, priority=PRIO_HIGH)) # wait for all tasks to complete print(ResultSet(results=high + normal).join(timeout=TASK_EXPIRY_TIME)) results_high = ResultSet(results=high).join() results_normal = ResultSet(results=normal).join() # determine amount of earlier executed normal prio tasks last_high_prio = sorted(results_high)[-1] early_normal = len([r for r in results_normal if r <= last_high_prio]) # amount of normal priority tasks executed earlier then the last executed high prio task # should ideally be 0 but surely not be greater than the amount of concurrent worker threads # give or take 3 amounts to account for early/late task race conditions # ie: some interlaced normal prio tasks get picked up by a worker thread because there are # no high prio tasks available at that time assert early_normal < (3 * settings.CELERY_WORKER_CONCURRENCY)
def run(self, callable, data): # Clear Queue self.clear_tasks() time.sleep(1) # Create all distributed tasks in the queue print("Creating tasks") tasks = [callable.delay(datum) for datum in data] t = tqdm(total=len(tasks), unit="task") results = ResultSet(tasks, app=self.app) start_time = time.time() # Wait for all distributed tasks to finish last_completed = 0 while True: if time.time() - start_time > 3600: # Will happen every hour start_time = time.time() self.spawn_workers() # Restart all slaves try: if results.ready(): break completed = results.completed_count() t.update(completed - last_completed) last_completed = completed except Exception as e: time.sleep(10) pass time.sleep(1) t.update(results.completed_count() - last_completed) return self
def aggregate_result_set(self, agg, acc): """ Loop on a set of celery AsyncResults and update the accumulator by using the aggregation function. :param agg: the aggregation function, (acc, val) -> new acc :param acc: the initial value of the accumulator :returns: the final value of the accumulator """ if not self.results: return acc backend = current_app().backend amqp_backend = backend.__class__.__name__.startswith("AMQP") rset = ResultSet(self.results) for task_id, result_dict in rset.iter_native(): check_mem_usage() # warn if too much memory is used result = result_dict["result"] if isinstance(result, BaseException): raise result self.received.append(len(result)) acc = agg(acc, result.unpickle()) if amqp_backend: # work around a celery bug del backend._cache[task_id] return acc
def getHashes(roms): resultSet = ResultSet([]) for rom in roms: # Need the full path for the MD5 hash function to operate on the file fullPath = settings.ROMS_FOLDER + "/" + rom['filename'] resultSet.add(getMD5Hash.delay(fullPath)) return resultSet.get()
def __init__(self, config, pool, manager_conf, pick_conf, cleaner): super(manager, self).__init__(manager_conf, pick_conf) self.path = getenv('MW_HOME') assert self.path != None self.tasks = ResultSet([]) self.backoff = int(config['retry_backoff']) self.powlim = int(config['max_backoff_power']) # backend codes to retry # CONFIRM: we're not retrying with other codes codes = [] if config['retry_forever_list'] != '': codes += [int(c) for c in config['retry_forever_list'].split(',')] if config['retry_sometime_list'] != '': codes += [int(c) for c in config['retry_sometime_list'].split(',')] self.backend_retry = set(codes) # thresholds self.thresholds = defaultdict(lambda: {}) self.thresholds['audio']['score'] = int(config["fp_audio_score"]) self.thresholds['video']['score'] = int(config["fp_video_score"]) self.thresholds['audio']['duration'] = int(config["fp_audio_duration"]) self.thresholds['video']['duration'] = int(config["fp_video_duration"]) self.task_set_join_timeout = int(config['task_set_join_timeout']) self.pool = pool self.cleaner = cleaner self.taskm = defaultdict(dict)
def aggregate_result_set(self, agg, acc): """ Loop on a set of celery AsyncResults and update the accumulator by using the aggregation function. :param agg: the aggregation function, (acc, val) -> new acc :param acc: the initial value of the accumulator :returns: the final value of the accumulator """ if isinstance(self.oqtask, types.FunctionType): # don't use celery return super(OqTaskManager, self).aggregate_result_set(agg, acc) if not self.results: return acc backend = current_app().backend amqp_backend = backend.__class__.__name__.startswith('AMQP') rset = ResultSet(self.results) for task_id, result_dict in rset.iter_native(): idx = self.task_ids.index(task_id) self.task_ids.pop(idx) parallel.check_mem_usage() # warn if too much memory is used result = result_dict['result'] if isinstance(result, BaseException): raise result self.received.append(len(result)) acc = agg(acc, result.unpickle()) if amqp_backend: # work around a celery bug del backend._cache[task_id] return acc
def test_iterate_respects_subpolling_interval(self): r1 = self.app.AsyncResult(uuid()) r2 = self.app.AsyncResult(uuid()) backend = r1.backend = r2.backend = Mock() backend.subpolling_interval = 10 ready = r1.ready = r2.ready = Mock() def se(*args, **kwargs): ready.side_effect = KeyError() return False ready.return_value = False ready.side_effect = se x = ResultSet([r1, r2]) with self.dummy_copy(): with patch('celery.result.time') as _time: with self.assertRaises(KeyError): list(x.iterate()) _time.sleep.assert_called_with(10) backend.subpolling_interval = 0 with patch('celery.result.time') as _time: with self.assertRaises(KeyError): ready.return_value = False ready.side_effect = se list(x.iterate()) self.assertFalse(_time.sleep.called)
def aggregate_result_set(self, agg, acc): """ Loop on a set of celery AsyncResults and update the accumulator by using the aggregation function. :param agg: the aggregation function, (acc, val) -> new acc :param acc: the initial value of the accumulator :returns: the final value of the accumulator """ if isinstance(self.oqtask, types.FunctionType): # don't use celery return super(OqTaskManager, self).aggregate_result_set( agg, acc) if not self.results: return acc backend = current_app().backend amqp_backend = backend.__class__.__name__.startswith('AMQP') rset = ResultSet(self.results) for task_id, result_dict in rset.iter_native(): idx = self.task_ids.index(task_id) self.task_ids.pop(idx) parallel.check_mem_usage() # warn if too much memory is used result = result_dict['result'] if isinstance(result, BaseException): raise result self.received.append(len(result)) acc = agg(acc, result.unpickle()) if amqp_backend: # work around a celery bug del backend._cache[task_id] return acc
def test_result_set_error(self, manager): assert manager.inspect().ping() rs = ResultSet([raise_error.delay(), add.delay(1, 1)]) rs.get(timeout=TIMEOUT, propagate=False) assert rs.results[0].failed() assert rs.results[1].successful()
def test_result_set_error(self, manager): assert_ping(manager) rs = ResultSet([raise_error.delay(), add.delay(1, 1)]) rs.get(timeout=TIMEOUT, propagate=False) assert rs.results[0].failed() assert rs.results[1].successful()
def test_result_set_error(self, manager): assert list(manager.inspect().ping().values())[0] == {"ok": "pong"} rs = ResultSet([raise_error.delay(), add.delay(1, 1)]) rs.get(timeout=TIMEOUT, propagate=False) assert rs.results[0].failed() assert rs.results[1].successful()
def test_times_out(self): r1 = self.app.AsyncResult(uuid) r1.ready = Mock() r1.ready.return_value = False x = ResultSet([r1]) with self.dummy_copy(): with patch('celery.result.time'): with self.assertRaises(TimeoutError): list(x.iterate(timeout=1))
def test_times_out(self): r1 = self.app.AsyncResult(uuid) r1.ready = Mock() r1.ready.return_value = False x = ResultSet([r1]) with self.dummy_copy(): with patch("celery.result.time"): with self.assertRaises(TimeoutError): list(x.iterate(timeout=1))
def run(self, expt_group, arch, dataset, metric, log_interval): for s in reversed(range(self.s_max + 1)): # initial number of configurations n = int(ceil(self.B / self.max_iter / (s + 1) * (self.eta**s))) # initial number of iterations to run configurations for r = self.max_iter * self.eta**(-s) # Successive Halving with (n, r) T = [ self.get_random_hyperparameter_configuration() for _ in range(n) ] for i in range(s + 1): # Run each of the n_i configs for r_i iters and keep best n_i / eta n_i = n * self.eta**(-i) r_i = int(r * self.eta**i) self.logger.info('%s configurations, %s iterations each', len(T), r_i) result_set = [] for t in T: for k, v in self.fixed_args.items(): t[k] = v async_res = self.run_then_return_eval( r_i, expt_group, arch, dataset, log_interval, t) result_set.append(async_res) result_set = ResultSet(result_set) result_set.join_native() self.logger.info('Completed new round') results = [] for async_res in result_set: res = async_res.get() result = { 'lr': res['args']['lr'], 'filters': res['args']['holistic_filters'], 'reg': res['args']['regularization'] } result[metric] = res['dev'][metric] results.append(result) if result[metric] > self.best_metric: self.best_metric = result[metric] self.logger.info('This is a new best...') sorted_results = sorted(results, key=lambda d: d[metric], reverse=True) for d in sorted_results: d.pop(metric) T = sorted_results[:int(n_i) // self.eta]
def launch_elasticsearch(): from celery_routing.elasticsearch.tasks import index_users, index_tweets print('Launching indexing') rs = ResultSet([ background(index_users), background(index_tweets), ]) print(rs.join()) print('Done')
def test_add_discard(self): x = ResultSet([]) x.add(AsyncResult("1")) self.assertIn(AsyncResult("1"), x.results) x.discard(AsyncResult("1")) x.discard(AsyncResult("1")) x.discard("1") self.assertNotIn(AsyncResult("1"), x.results) x.update([AsyncResult("2")])
def test_add_discard(self): x = ResultSet([]) x.add(AsyncResult('1')) self.assertIn(AsyncResult('1'), x.results) x.discard(AsyncResult('1')) x.discard(AsyncResult('1')) x.discard('1') self.assertNotIn(AsyncResult('1'), x.results) x.update([AsyncResult('2')])
def testbuf(padbytes=0, megabytes=0): padding = float(padbytes) + 2**20 * float(megabytes) results = [] print('> padding: %r' % (padding, )) for i in range(8 * 4): results.append(sleeping.delay(1, kw='x' * int(padding))) time.sleep(0.01) res = ResultSet(results) print(res.join())
def testbuf(padbytes=0, megabytes=0): padding = float(padbytes) + 2 ** 20 * float(megabytes) results = [] print('> padding: %r' % (padding, )) for i in range(8 * 4): results.append(sleeping.delay(1, kw='x' * int(padding))) time.sleep(0.01) res = ResultSet(results) print(res.join())
def end(self): """Do the push once all results are in""" dbg("Waiting for %d celery task(s)..." % len(self.results)) result_set = ResultSet(self.results) results = result_set.get( timeout=self.context.get("timeout", None), propagate=self.context.get("propagate", True), interval=self.context.get("interval", 0.5), ) result_set.forget() if results and self.context.get("flatten", False): results = flatten(results) self.push(results)
def convert_file(is_video, input_path, output_format, video_length, part_length): rs = ResultSet([]) for i in range(get_total_workers()): start_at = i * part_length stop_at = start_at + part_length if i != get_total_workers( ) - 1 else video_length print("worker {} will process from {}s to {}s".format( i + 1, start_at, stop_at)) rs.add( process_part.delay(is_video, input_path, output_format, start_at, stop_at)) return rs.get()
def run_task(self, *args, **options): # try to compose task if not specified if not self.task: self.task = self.compose(*args, **options) # execute task based on selected method if options['method'] in ['sync', 'async']: # verify if broker is accessible (eg: might not be started in dev. environment) try: app.connection().ensure_connection(max_retries=3) except kombu.exceptions.OperationalError: log.warning( 'Connection with task broker %s unavailable, tasks might not be starting.', settings.CELERY_BROKER_URL) task_id = self.task.apply_async(args=self.args, kwargs=self.kwargs) log.info('Task scheduled for execution.') log.debug("Task ID: %s", task_id.id) # wrap a single task in a resultset to not have 2 ways to handle results if not isinstance(task_id, ResultSet): task_id = ResultSet([task_id]) if options['method'] == 'sync': return self.wait_for_result(task_id) else: # if async return taskid to allow query for status later on return [r.id for r in task_id.results] else: # By default execute the task directly without involving celery or a broker. # Return all results without raising exceptions. log.info('Executing task directly.') return self.task.apply(*self.args, **self.kwargs).get(propagate=False)
def saveResultsAndCleanUp(self): """ Executes after the retrieval is done. """ if self.use_celery: print("Waiting for tasks to complete...") res=ResultSet(self.tasks) while not res.ready(): try: time.sleep(7) except KeyboardInterrupt: print("Cancelled waiting") break print("All tasks finished.") for writer in self.writers: self.writers[writer].saveAsJSON(os.path.join(self.exp["exp_dir"],self.writers[writer].table_name+".json"))
def launch_tasks(): print('Launching different types of tasks') r1 = background(tasks.fast_task, x=5, y=10) r1.then(report) r2 = background(tasks.long_task, 42) r2.then(report) r3 = background(tasks.default_task) r3.then(report) r4 = background(tasks.elasticsearch_task) r4.then(report) rs = ResultSet([r1, r2, r3, r4]) rs.join() print('Done')
def get_result(request): ret = { 'status': 'error', 'result': '', 'messages': [ '', ], } if request.method == 'POST' and request.user: try: user = MDBUser.objects.get(username=request.user.username) except Exception, e: ret['messages'][0] = "<strong>FATAL</strong>(get_result.user): %s" % e else: # Note: this is NOT status of tasks, 'success' here means that # get_result() request was processed correctly ret['status'] = 'success' async_res = AsyncResult(request.POST['task_id']) if async_res.ready(): # Get all subtasks spawned by parent subtasks = None #ust_get_ids(user) # Create list of AsyncResults from list of task_ids async_results = [] for task_id in subtasks: async_results.append(AsyncResult(task_id)) # And also ResultSet for convenience async_res_set = ResultSet(async_results) ret['messages'][0] = 'parent task %s: %d of %d subtasks completed' %\ (request.POST['task_id'][:8], async_res_set.completed_count(), async_res_set.total, ) # All tasks completed ? if async_res_set.ready(): # All tasks done, forget about those task ids #ust_clear_ids(user) # Any of them failed ? if async_res_set.failed(): ret['result'] = 'FAILURE' for async_res in async_results: if async_res.state == 'FAILURE': ret['messages'].append("<strong>ERROR</strong>(get_result.FAILURE): '%s':'%s'" %\ (async_res.task_id[:8], async_res.result, )) else: ret['result'] = 'SUCCESS' else: ret['result'] = 'PENDING' else: ret['result'] = 'PENDING' ret['messages'][0] = 'parent task %s: PENDING' % \ (request.POST['task_id'], )
def saveResultsAndCleanUp(self): """ Executes after the retrieval is done. """ if self.use_celery: print("Waiting for tasks to complete...") res = ResultSet(self.tasks) while not res.ready(): try: time.sleep(7) except KeyboardInterrupt: print("Cancelled waiting") break print("All tasks finished.") for writer in self.writers: self.writers[writer].saveAsJSON( os.path.join(self.exp["exp_dir"], self.writers[writer].table_name + ".json"))
def aggregate_result_set(self, agg, acc): """ Loop on a set results and update the accumulator by using the aggregation function. :param agg: the aggregation function, (acc, val) -> new acc :param acc: the initial value of the accumulator :returns: the final value of the accumulator """ if not self.results: return acc distribute = oq_distribute() # not called for distribute == 'no' if distribute == 'celery': backend = current_app().backend amqp_backend = backend.__class__.__name__.startswith('AMQP') rset = ResultSet(self.results) for task_id, result_dict in rset.iter_native(): idx = self.task_ids.index(task_id) self.task_ids.pop(idx) check_mem_usage() # warn if too much memory is used result = result_dict['result'] if isinstance(result, BaseException): raise result self.received.append(len(result)) acc = agg(acc, result.unpickle()) if amqp_backend: # work around a celery bug del backend._cache[task_id] return acc elif distribute == 'futures': for future in as_completed(self.results): check_mem_usage() # log a warning if too much memory is used result = future.result() if isinstance(result, BaseException): raise result self.received.append(len(result)) acc = agg(acc, result.unpickle()) return acc
def test_get(self): x = ResultSet(map(AsyncResult, [1, 2, 3])) b = x.results[0].backend = Mock() b.supports_native_join = False x.join_native = Mock() x.join = Mock() x.get() self.assertTrue(x.join.called) b.supports_native_join = True x.get() self.assertTrue(x.join_native.called)
def obj_get_list(self, bundle, **kwargs): query = bundle.request.GET.get('q') if not query: response = {'status': 0, 'message': 'Empty query'} else: #from my_task.tasks import google, duck_duck_go, twitter from my_task.tasks import google, duck_duck_go, twitter # Async process from celery.result import ResultSet #A collection of results. rs = ResultSet([]) # Add AsyncResult as a new member of the set. rs.add(google.delay(query)) rs.add(duck_duck_go.delay(query)) rs.add(twitter.delay(query)) response = rs.get() # waiting for the results url = "http://127.0.0.1:8000/my_resources/v1/search/?q={query}".format( query=query) try: response = { 'query': query, 'results': { 'google': { 'text': response[0], 'url': url }, 'duckduckgo': { 'text': response[1], 'url': url }, 'twitter': { 'text': response[2], 'url': url } } } except AttributeError: response = {'status': 0, 'message': 'Result Timeout'} # For immediate response raise ImmediateHttpResponse(response=HttpCreated( content=json.dumps(response), content_type='application/json; charset=UTF-8'))
def _iterfutures(self): # compatibility wrapper for different concurrency frameworks if self.distribute == 'no': for result in self.results: yield mkfuture(result) elif self.distribute == 'celery': rset = ResultSet(self.results) for task_id, result_dict in rset.iter_native(): idx = self.task_ids.index(task_id) self.task_ids.pop(idx) fut = mkfuture(result_dict['result']) # work around a celery bug del app.backend._cache[task_id] yield fut else: # future interface for fut in as_completed(self.results): yield fut
def aggregate_result_set(self, agg, acc): """ Loop on a set of celery AsyncResults and update the accumulator by using the aggregation function. :param agg: the aggregation function, (acc, val) -> new acc :param acc: the initial value of the accumulator :returns: the final value of the accumulator """ if not self.results: return acc backend = current_app().backend rset = ResultSet(self.results) for task_id, result_dict in rset.iter_native(): check_mem_usage() # log a warning if too much memory is used result = result_dict['result'] if isinstance(result, BaseException): raise result acc = agg(acc, result.unpickle()) del backend._cache[task_id] # work around a celery bug return acc
def saveResultsAndCleanUp(self): """ Executes after the retrieval is done. Should the results be saved? """ # super().saveResultsAndCleanUp() if self.use_celery: print("Waiting for tasks to complete...") res = ResultSet(self.tasks) while not res.ready(): try: time.sleep(7) except KeyboardInterrupt: print("Cancelled waiting") break print("All tasks finished.") if self.options.get("list_missing_files", False): self.saveMissingFiles()
def monitor_cluster_cpu(nodes): cpu_tasks = [] all_cpu_util = [] cpu_util = 0 for node in nodes: cpu_tasks.append(cpu_monitor.delay()) all_cpu_util = ResultSet(cpu_tasks).join_native() cpu_util = sum(ResultSet)/nodes print all_cpu_util, cpu_util threading.Timer(60, monitor_cluster_cpu).start()
def aggregate_result_set(self, agg, acc): """ Loop on a set of celery AsyncResults and update the accumulator by using the aggregation function. :param agg: the aggregation function, (acc, val) -> new acc :param acc: the initial value of the accumulator :returns: the final value of the accumulator """ if not self.results: return acc backend = current_app().backend rset = ResultSet(self.results) for task_id, result_dict in rset.iter_native(): check_mem_usage() # warn if too much memory is used result = result_dict['result'] if isinstance(result, BaseException): raise result self.received += len(result) acc = agg(acc, result.unpickle()) del backend._cache[task_id] # work around a celery bug return acc
def think(entry_id, services, domain_uri): # get Face++ group name if service is turned on group_name = None if services.get('facepp'): group_name = get_fb_user(services['facepp']).id # async get keywords from VoiceBase # only do this for async mode if services.get('voicebase') and settings.USE_ASYNC: save_service_status(entry_id, 'VOICEBASE', 'PROGRESS') generate_voice_keyword_tags.delay(entry_id) # get image samplings from Kaltura video images = generate_image_samplings_from_kaltura(entry_id) # process object recognition # process facial recognition is_generate_object_tags = services.get('stockpodium', False) is_generate_friend_tags = services.get('facepp', False) if is_generate_object_tags: save_service_status(entry_id, 'STOCKPODIUM', 'PROGRESS') if is_generate_friend_tags: save_service_status(entry_id, 'FACEPP', 'PROGRESS') results = [] for i in images: image_url = urlparse.urljoin(domain_uri, settings.MEDIA_URL + i) if is_generate_object_tags: if settings.USE_ASYNC: results.append(generate_object_tags.delay(entry_id, image_url)) else: generate_object_tags(entry_id, image_url) if is_generate_friend_tags: if settings.USE_ASYNC: results.append(generate_friend_tags.delay(entry_id, image_url, group_name)) else: generate_friend_tags(entry_id, image_url, group_name) # wait for tagging to be complete if results: ResultSet(results).join() save_service_status(entry_id, 'STOCKPODIUM', 'SUCCESS') save_service_status(entry_id, 'FACEPP', 'SUCCESS') # clean generate image samplings for i in images: f = os.path.join(settings.MEDIA_ROOT, i) if os.path.isfile(f): os.unlink(f)
def distributetasks(self, dnsublist, targetdnlist, workername=""): # Send the Task to Celery Queue import ltecpxx.mrosimpleexecutor as r taskid = uuid.uuid1().int # create a unique main task id self.tasklist.append({taskid: ''}) resultset = ResultSet([]) for sourcedns in dnsublist: # send sub tasks for the main task to Celery args = (sourcedns, targetdnlist) kwargs= {'workername':workername} celery = Celery() celery.conf.CELERY_ALWAYS_EAGER = True celery.conf.CELERY_ALWAYS_EAGER = True result = r.doTargetprefilter.apply_async(args,kwargs) resultset.add(result) #print("Result Is Done %s Value is %d Is Done Now %s" % (result.ready(),result.get(),result.ready())) print("Result Is Done %s " % result.ready()) self.tasklist[-1][taskid] = resultset print("Task List Conents", self.tasklist) # return the status of of the operation resp = {'TaskId': taskid, 'NumberofSubtasks': dnsublist.__len__(), 'RunState': str(RunState.Submitted)} return resp
def generate_image_samplings_from_kaltura(entry_id): duration = None # duration in seconds attempts = 5 while attempts > 0: data = get_entry_metadata(entry_id) if 'duration' in data: duration = data['duration'] break time.sleep(60.0 / attempts) attempts -= 1 if duration is None: raise Exception('Cannot find video on Kaltura: "%s"' % entry_id) # number of samples should be proportion to the video duration step = min(1, max(5, int(duration / 5.0))) if settings.USE_ASYNC: results = ResultSet([generate_thumbnail_at_time_from_kaltura.delay(entry_id, i) for i in range(0, duration, step)]) return results.join() else: return [generate_thumbnail_at_time_from_kaltura(entry_id, i) for i in range(0, duration, step)]
def initialise_fb_user(domain_uri, access_token): fb_user = get_fb_user(access_token) group_name = fb_user.id photos = get_fb_photos(access_token) if settings.USE_ASYNC: results = ResultSet([process_fb_photo.delay(d, access_token) for d in photos['data']]) processed_photos = [p for photos in results.join() for p in photos] else: processed_photos = [process_fb_photo(d, access_token) for d in photos['data']] processed_photos = [p for photos in processed_photos for p in photos] filtered_photos = filter_fb_photos_for_training(processed_photos) media_uri = urlparse.urljoin(domain_uri, settings.MEDIA_URL) if settings.USE_ASYNC: results = ResultSet([upload_fb_photos_for_training.delay([p], group_name, media_uri) for p in filtered_photos]) results.join() else: upload_fb_photos_for_training(filtered_photos, group_name, media_uri) train_fb_photos(group_name)
def _iter_celery(self, backurl=None): results = [] for piks in self._genargs(backurl): res = safetask.delay(self.task_func, piks) # populating Starmap.task_ids, used in celery_cleanup self.task_ids.append(res.task_id) results.append(res) yield len(results) for task_id, result_dict in ResultSet(results).iter_native(): self.task_ids.remove(task_id) if CELERY_RESULT_BACKEND.startswith('rpc:'): # work around a celery/rabbitmq bug del app.backend._cache[task_id] yield result_dict['result']
def divide_work(start_angle, stop_angle, num_angles, num_nodes, levels, NACA, num_samples, viscosity, speed, sim_time): anglediff = (stop_angle - start_angle) // num_angles # http://docs.celeryproject.org/en/latest/reference/celery.result.html#celery.result.ResultSet in_progress = ResultSet([]) results = [] # +1 to match runme.sh for i in range(0, num_angles + 1): angle = start_angle + anglediff * i key = _key_string(angle, num_nodes, levels, NACA, num_samples, viscosity, speed, sim_time) # query database # possibly add something more to check if the result is queued and pending. Simultaneous queries will cause double tasks to be added to the queue, since the tasks are pending. Possibly add a global queue that we append in progress results to, which we check if status is pending. result = airfoil.AsyncResult(key) if result.status == 'SUCCESS': alrdy_queued = True results.append(result.get()) elif result.status in ['STARTED', 'RETRY']: alrdy_queued = True in_progress.add(result) elif result.status == 'PENDING': alrdy_queued = False else: print('Task status FAILURE.') alrdy_queued = False if not alrdy_queued: result = airfoil.apply_async( (angle, num_nodes, levels, NACA, num_samples, viscosity, speed, sim_time, key), task_id=key) in_progress.add(result) # waiting for all results # in_progress.join_native() is supposed to be more efficient, but it seems to hang results.extend(in_progress.join()) # list of all results result_file = _result_string(start_angle, stop_angle, num_angles, num_nodes, levels, NACA, num_samples, viscosity, speed, sim_time) results_archive = '/home/ubuntu/results/' + result_file tar_cmd = [ 'tar', '-zcvf', results_archive, '-C', '/home/ubuntu/sync_results' ] tar_cmd.extend(results) subprocess.call(tar_cmd) return result_file
class manager(server): def __init__(self, config, pool, manager_conf, pick_conf, cleaner): super(manager, self).__init__(manager_conf, pick_conf) self.path = getenv('MW_HOME') assert self.path != None self.tasks = ResultSet([]) self.backoff = int(config['retry_backoff']) self.powlim = int(config['max_backoff_power']) # backend codes to retry # CONFIRM: we're not retrying with other codes codes = [] if config['retry_forever_list'] != '': codes += [int(c) for c in config['retry_forever_list'].split(',')] if config['retry_sometime_list'] != '': codes += [int(c) for c in config['retry_sometime_list'].split(',')] self.backend_retry = set(codes) # thresholds self.thresholds = defaultdict(lambda: {}) self.thresholds['audio']['score'] = int(config["fp_audio_score"]) self.thresholds['video']['score'] = int(config["fp_video_score"]) self.thresholds['audio']['duration'] = int(config["fp_audio_duration"]) self.thresholds['video']['duration'] = int(config["fp_video_duration"]) self.task_set_join_timeout = int(config['task_set_join_timeout']) self.pool = pool self.cleaner = cleaner self.taskm = defaultdict(dict) @catch_and_die('mwtm_manager') def run(self): while True: self.loop() @staticmethod def record(t, pid=0): yield db_execute(BEGIN_QUERY, pid, t.id) #c, r = yield db_insert(QUERY_EVENT, t.uuid, pid) #if c <= 0: # yield db_result() #else: # yield db_result(r) def one_request(self, wait=False): with self.req_cond: return self.reqq.get(wait) def task_check(self, t, accs, backends): if not t.account in accs: self.logger.debug("account %s maybe deleted" % t.account) return False for b in accs[t.account].backends: if not b.backend in backends: self.logger.warning('backend %s for account %s ' + 'inconsistent with backends in db', b.backend, t.account) return False return True def buf_tasks(self, reqs): accs = self.accounts() backends = self.backends() for t in reqs: try: self.logger.info("receivce task from picker, task_uuid: %s, " "site_asset_id: %s" % (t.uuid, t.site_asset_id)) self.logger.debug("receive task info:%s" % t._asdict()) g_logger.info(trans2json(message="site_asset_id:%s, " "task_uuid:%s, external_id:%s" % (t.site_asset_id, t.uuid, t.external_id), action="receive picked task")) if not self.task_check(t, accs, backends): self.reply(t) continue acc = accs[t.account]._asdict() acc["backends"] = [v._asdict() for v in acc["backends"]] backs = {} for k, v in backends.iteritems(): backs[k] = v._asdict() self.logger.debug("add task's account: %s, backends: %s" % (acc, backs)) ct = Task().query.delay(t._asdict(), acc, backs) self.taskm[ct.task_id]['celery_task'] = ct self.taskm[ct.task_id]['task'] = t self.tasks.add(ct) self.logger.info("add task to celery, task_uuid: %s, " "site_asset_id: %s, celery_uuid: %s " % \ (t.uuid, t.site_asset_id, ct.task_id)) g_logger.info(trans2json(message="site_asset_id:%s, " "task_uuid:%s, external_id:%s" % (t.site_asset_id, t.uuid, t.external_id), action="add task to celery")) except Exception, ex: self.reply(t) self.logger.error("catch exception from buf tasks, " "task_uuid: %s , site_asset_id: %s" % (t.uuid, t.site_asset_id), exc_info=True) continue try: db_txn(self.pool, partial(self.record, t)) except Exception: self.logger.error("failed to record execution for task %s" % t.uuid)
def test_takes_app_from_first_task(self): x = ResultSet([self.app.AsyncResult('id1')]) assert x.app is x.results[0].app x.app = self.app assert x.app is self.app
def gather_scrub_work(): logger.info("Divide scrubbing work among allowed Storage Routers") scrub_locations = {} for storage_driver in StorageDriverList.get_storagedrivers(): for partition in storage_driver.partitions: if DiskPartition.ROLES.SCRUB == partition.role: logger.info( "Scrub partition found on Storage Router {0}: {1}".format(storage_driver.name, partition.folder) ) if storage_driver.storagerouter not in scrub_locations: try: _ = SSHClient(storage_driver.storagerouter.ip) scrub_locations[storage_driver.storagerouter] = str(partition.path) except UnableToConnectException: logger.warning("StorageRouter {0} is not reachable".format(storage_driver.storagerouter.ip)) if len(scrub_locations) == 0: raise RuntimeError("No scrub locations found") vdisk_guids = set() for vmachine in VMachineList.get_customer_vmachines(): for vdisk in vmachine.vdisks: if vdisk.info["object_type"] in ["BASE"] and len(vdisk.child_vdisks) == 0: vdisk_guids.add(vdisk.guid) for vdisk in VDiskList.get_without_vmachine(): if vdisk.info["object_type"] in ["BASE"] and len(vdisk.child_vdisks) == 0: vdisk_guids.add(vdisk.guid) logger.info("Found {0} virtual disks which need to be check for scrub work".format(len(vdisk_guids))) local_machineid = System.get_my_machine_id() local_scrub_location = None local_vdisks_to_scrub = [] result_set = ResultSet([]) storage_router_list = [] for index, scrub_info in enumerate(scrub_locations.items()): start_index = index * len(vdisk_guids) / len(scrub_locations) end_index = (index + 1) * len(vdisk_guids) / len(scrub_locations) storage_router = scrub_info[0] vdisk_guids_to_scrub = list(vdisk_guids)[start_index:end_index] local = storage_router.machine_id == local_machineid logger.info( "Executing scrub work on {0} Storage Router {1} for {2} virtual disks".format( "local" if local is True else "remote", storage_router.name, len(vdisk_guids_to_scrub) ) ) if local is True: local_scrub_location = scrub_info[1] local_vdisks_to_scrub = vdisk_guids_to_scrub else: result_set.add( ScheduledTaskController._execute_scrub_work.s( scrub_location=scrub_info[1], vdisk_guids=vdisk_guids_to_scrub ).apply_async(routing_key="sr.{0}".format(storage_router.machine_id)) ) storage_router_list.append(storage_router) logger.info("Launched scrub task on Storage Router {0}".format(storage_router.name)) # Remote tasks have been launched, now start the local task and then wait for remote tasks to finish if local_scrub_location is not None and len(local_vdisks_to_scrub) > 0: ScheduledTaskController._execute_scrub_work( scrub_location=local_scrub_location, vdisk_guids=local_vdisks_to_scrub ) all_results = result_set.join( propagate=False ) # Propagate False makes sure all jobs are waited for even when 1 or more jobs fail for index, result in enumerate(all_results): if result is not None: logger.error( "Scrubbing failed on Storage Router {0} with error {1}".format( storage_router_list[index].name, result ) )
def test_clear(self): x = ResultSet([]) r = x.results x.clear() self.assertIs(x.results, r)
def compile_and_copy(self, design_name, hashes, jobinfo, userjobconfig): if userjobconfig.enableEMAIL: email_start(userjobconfig, jobinfo, design_name, self.request.hostname) rs = ResultSet([]) rl = RedisLogger(design_name, jobinfo, userjobconfig.logging_on) rl2 = RedisLoggerStream(design_name, jobinfo, userjobconfig.logging_on) #isfbox = re.match("^f[0-9][0-9]+", self.request.hostname) #isfbox = isfbox is not None base_dir = "/scratch/" #if isfbox: # base_dir = "/data/" # create scratch space on this node for compiling the design, then clone # for now, do not delete the scratch space, just keep making new ones # 1) preserve work dir for debugging # 2) let a user run multiple jobs at once design_dir = base_dir + userjobconfig.username + '/celery-temp/' + jobinfo + "/" + design_name # remove old results for that design if they exist #rl.local_logged('rm -rf ' + design_dir) rl.local_logged('mkdir -p ' + design_dir) with lcd(design_dir): rl.local_logged('git clone ' + userjobconfig.rocket_chip_location + " rocket-chip") rc_dir = design_dir + '/rocket-chip' with lcd(rc_dir): # checkout the correct hash rl.local_logged('git checkout ' + hashes['rocket-chip']) rl.local_logged('git submodule update --init') # copy designs scala file configs_dir = 'src/main/scala/config' rl.local_logged('mkdir -p ' + configs_dir) rl.local_logged('cp ' + userjobconfig.distribute_rocket_chip_loc + '/' + jobinfo + '/' + userjobconfig.CONF + '.scala ' + configs_dir + '/') with lcd(rc_dir + '/vlsi'): rl.local_logged('git submodule update --init --recursive') # now, apply patches apply_recursive_patches(userjobconfig.distribute_rocket_chip_loc + '/' + jobinfo + '/patches', rc_dir) # at this point, design_dir/rocket-chip will contain everything we need to # do the various compiles shell_env_args_conf = copy(userjobconfig.shell_env_args) shell_env_args_conf['CONFIG'] = design_name cpp_emu_name = 'emulator-' + userjobconfig.MODEL + '-' + design_name vsim_emu_name = 'simv-' + userjobconfig.MODEL + '-' + design_name # make C++ emulator # NOTE: This is currently required to get the dramsim2_ini directory # and get list of tests to run # TODO: do we need to get list of tests to run per environment? if 'emulator' in userjobconfig.tests: with lcd(rc_dir + '/emulator'), shell_env(**shell_env_args_conf): rl2.local_logged('make ' + cpp_emu_name + ' 2>&1') else: with lcd(rc_dir + '/emulator'), shell_env(**shell_env_args_conf), settings(warn_only=True): # even if emulator is broken, need dramsim rl2.local_logged('make ' + cpp_emu_name + ' 2>&1') with lcd(rc_dir + '/emulator'), shell_env(**shell_env_args_conf): rl.local_logged('cp -Lr ../emulator ' + userjobconfig.distribute_rocket_chip_loc + '/' + jobinfo + '/' + design_name + '/emulator/') testslist = read_tests(rc_dir + '/emulator/generated-src/', design_name) print("running tests:") print(testslist) """ Run C++ emulator """ if 'emulator' in userjobconfig.tests: for y in testslist: rs.add(emulatortest.apply_async([design_name, y, jobinfo, userjobconfig], queue='test')) """ Run vsim """ if 'vsim' in userjobconfig.tests: # make vsim, copy with lcd(rc_dir + '/vsim'), shell_env(**shell_env_args_conf), prefix('source ' + vlsi_bashrc): rl2.local_logged('make ' + vsim_emu_name + ' 2>&1') rl.local_logged('cp -Lr ../vsim ' + userjobconfig.distribute_rocket_chip_loc + '/' + jobinfo + '/' + design_name + '/vsim/') # copy dramsim2_ini directory for vsim with lcd(userjobconfig.distribute_rocket_chip_loc + '/' + jobinfo + '/' + design_name): rl.local_logged('cp -r emulator/emulator/dramsim2_ini vsim/vsim/') # start vsim tasks for y in testslist: rs.add(vsimtest.apply_async([design_name, y, jobinfo, userjobconfig], queue='test')) """ Run vcs-sim-rtl """ if 'vcs-sim-rtl' in userjobconfig.tests: # make vcs-sim-rtl, copy with lcd(rc_dir + '/vlsi/vcs-sim-rtl'), shell_env(**shell_env_args_conf), prefix('source ' + vlsi_bashrc): rl2.local_logged('make ' + vsim_emu_name + ' 2>&1') rl.local_logged('cp -Lr ../vcs-sim-rtl ' + userjobconfig.distribute_rocket_chip_loc + '/' + jobinfo + '/' + design_name + '/vcs-sim-rtl/') # copy dramsim2_ini directory for vcs-sim-rtl with lcd(userjobconfig.distribute_rocket_chip_loc + '/' + jobinfo + '/' + design_name): rl.local_logged('cp -r emulator/emulator/dramsim2_ini vcs-sim-rtl/vcs-sim-rtl/') for y in testslist: rs.add(vcs_sim_rtl_test.apply_async([design_name, y, jobinfo, userjobconfig], queue='test')) """ run dc-syn """ if 'dc-syn' in userjobconfig.tests: with lcd(rc_dir + '/vlsi'), shell_env(**shell_env_args_conf), prefix('source ' + vlsi_bashrc): rl2.local_logged('make dc 2>&1') # vlsi, dc with lcd(rc_dir + '/vlsi/dc-syn'), shell_env(**shell_env_args_conf), prefix('source ' + vlsi_bashrc): # TODO: what does -jN do here? #rl2.local_logged('make 2>&1') rl.local_logged('cp -r current-dc/reports ' + userjobconfig.distribute_rocket_chip_loc + '/' + jobinfo + '/' + design_name + '/dc-syn/') rl.local_logged('cp -r current-dc/results ' + userjobconfig.distribute_rocket_chip_loc + '/' + jobinfo + '/' + design_name + '/dc-syn/') rl.clear_log() # if we made it this far, clear the redis log list if 'vcs-sim-gl-syn' in userjobconfig.tests: with lcd(rc_dir + '/vlsi/vcs-sim-gl-syn'), shell_env(**shell_env_args_conf), prefix('source ' + vlsi_bashrc): # todo actually use the name rl2.local_logged('make 2>&1') # todo copy rl.local_logged('cp -Lr ../vcs-sim-gl-syn ' + userjobconfig.distribute_rocket_chip_loc + '/' + jobinfo + '/' + design_name + '/vcs-sim-gl-syn/') # copy dramsim2_ini directory for vcs-sim-gl-syn with lcd(userjobconfig.distribute_rocket_chip_loc + '/' + jobinfo + '/' + design_name): rl.local_logged('cp -r emulator/emulator/dramsim2_ini vcs-sim-gl-syn/vcs-sim-gl-syn/') for y in testslist: rs.add(vcs_sim_gl_syn_test.apply_async([design_name, y, jobinfo, userjobconfig], queue='test')) """ run icc-par """ if 'icc-par' in userjobconfig.tests: with lcd(rc_dir + '/vlsi'), shell_env(**shell_env_args_conf), prefix('source ' + vlsi_bashrc): rl2.local_logged('make icc 2>&1') # vlsi, icc with lcd(rc_dir + '/vlsi/icc-par'), shell_env(**shell_env_args_conf), prefix('source ' + vlsi_bashrc): # TODO: what does -jN do here? #rl2.local_logged('make 2>&1') rl.local_logged('cp -r current-icc/reports ' + userjobconfig.distribute_rocket_chip_loc + '/' + jobinfo + '/' + design_name + '/icc-par/') rl.local_logged('cp -r current-icc/results ' + userjobconfig.distribute_rocket_chip_loc + '/' + jobinfo + '/' + design_name + '/icc-par/') if 'vcs-sim-gl-par' in userjobconfig.tests: with lcd(rc_dir + '/vlsi/vcs-sim-gl-par'), shell_env(**shell_env_args_conf), prefix('source ' + vlsi_bashrc): # todo actually use the name rl2.local_logged('make 2>&1') # todo copy rl.local_logged('cp -Lr ../vcs-sim-gl-par ' + userjobconfig.distribute_rocket_chip_loc + '/' + jobinfo + '/' + design_name + '/vcs-sim-gl-par/') # copy dramsim2_ini directory for vcs-sim-gl-par with lcd(userjobconfig.distribute_rocket_chip_loc + '/' + jobinfo + '/' + design_name): rl.local_logged('cp -r emulator/emulator/dramsim2_ini vcs-sim-gl-par/vcs-sim-gl-par/') for y in testslist: rs.add(vcs_sim_gl_par_test.apply_async([design_name, y, jobinfo, userjobconfig], queue='test')) rl.clear_log() # clear the redis log list return rs
def test_add(self): x = ResultSet([1]) x.add(2) self.assertEqual(len(x), 2) x.add(2) self.assertEqual(len(x), 2)
def test_takes_app_from_first_task(self): x = ResultSet([self.app.AsyncResult('id1')]) self.assertIs(x.app, x.results[0].app) x.app = self.app self.assertIs(x.app, self.app)
# sanity check that the riscv-tests hash is not the same as the riscv-tools hash # (this probably means that you forgot to do git submodule update --init inside riscv-tools) if hashes['riscv-tests'] == hashes['riscv-tools']: print(bcolors.FAIL + "riscv-tests hash matches riscv-tools hash. Did you forget to init the\nriscv-tests submodule?" + bcolors.ENDC) exit(1) with lcd(userjobconfig.install_dir + '/tests-installs'), shell_env(**userjobconfig.shell_env_args), settings(warn_only=True): local('git clone ' + userjobconfig.tests_location + ' ' + hashes['riscv-tests']) local('cd ' + hashes['riscv-tests'] + ' && git checkout ' + hashes['riscv-tests']) local('cd ' + hashes['riscv-tests'] + ' && git submodule update --init') local('cd ' + hashes['riscv-tests'] + '/isa && make -j32') local('cd ' + hashes['riscv-tests'] + '/benchmarks && make -j32') do_jackhammer() build_riscv_tests() compiles = ResultSet([]) for x in designs: compiles.add(compile_and_copy.apply_async([x, hashes, jobdirname, userjobconfig], queue='build')) print(bcolors.OKBLUE + "Your job has been launched. You can monitor it at fbox:8080" + bcolors.ENDC) print(bcolors.OKGREEN + "Your job id is " + jobdirname + bcolors.ENDC) # TODO generate job run report # 1 whether or not new tests/tools were installed # 2 where to find outputs # 3 how to use watch script # 4 jobid # 5 write it to file so that the watch script can use it
from celery_tasks import processFile from celery.result import ResultSet from sklearn.ensemble import RandomForestClassifier import pandas as pd #https://pypi.python.org/pypi/etaprogress/ from etaprogress.progress import ProgressBar print('--- Read training labels') train = pd.read_csv('./data/train_v2.csv') train_keys = dict([a[1] for a in train.iterrows()]) test_files = set(pd.read_csv('./data/sampleSubmission_v2.csv').file.values) print("--- Started processing") result = ResultSet([]) bar = ProgressBar(len(train)+len(test_files), max_width=40) #https://celery.readthedocs.org/en/latest/reference/celery.result.html#celery.result.ResultSet for k, filename in enumerate(list(train['file'])+list(test_files)): if filename in train_keys: result.add(processFile.delay(filename, train_keys[filename])) elif filename != "": result.add(processFile.delay(filename, 2)) #sponsored = train.loc[train['file'] == openfile] #if not sponsored.empty: #result.add(processFile.delay(openfile, data, int(sponsored['sponsored']))) #testing = sample.loc[sample['file'] == openfile] #if not testing.empty: #result.add(processFile.delay(openfile, data, int(sponsored['sponsored'])))
def gather_scrub_work(): """ Retrieve and execute scrub work :return: None """ logger.info('Gather Scrub - Started') scrub_locations = {} for storage_driver in StorageDriverList.get_storagedrivers(): for partition in storage_driver.partitions: if DiskPartition.ROLES.SCRUB == partition.role: logger.info('Gather Scrub - Storage Router {0:<15} has SCRUB partition at {1}'.format(storage_driver.storagerouter.ip, partition.path)) if storage_driver.storagerouter not in scrub_locations: try: _ = SSHClient(storage_driver.storagerouter) scrub_locations[storage_driver.storagerouter] = str(partition.path) except UnableToConnectException: logger.warning('Gather Scrub - Storage Router {0:<15} is not reachable'.format(storage_driver.storagerouter.ip)) if len(scrub_locations) == 0: raise RuntimeError('No scrub locations found') vdisk_guids = set() for vmachine in VMachineList.get_customer_vmachines(): for vdisk in vmachine.vdisks: if vdisk.info['object_type'] == 'BASE': vdisk_guids.add(vdisk.guid) for vdisk in VDiskList.get_without_vmachine(): if vdisk.info['object_type'] == 'BASE': vdisk_guids.add(vdisk.guid) logger.info('Gather Scrub - Checking {0} volumes for scrub work'.format(len(vdisk_guids))) local_machineid = System.get_my_machine_id() local_storage_router = None local_scrub_location = None local_vdisks_to_scrub = [] result_set = ResultSet([]) storage_router_list = [] for index, scrub_info in enumerate(scrub_locations.items()): start_index = index * len(vdisk_guids) / len(scrub_locations) end_index = (index + 1) * len(vdisk_guids) / len(scrub_locations) storage_router = scrub_info[0] vdisk_guids_to_scrub = list(vdisk_guids)[start_index:end_index] local = storage_router.machine_id == local_machineid logger.info('Gather Scrub - Storage Router {0:<15} ({1}) - Scrubbing {2} virtual disks'.format(storage_router.ip, 'local' if local is True else 'remote', len(vdisk_guids_to_scrub))) if local is True: local_storage_router = storage_router local_scrub_location = scrub_info[1] local_vdisks_to_scrub = vdisk_guids_to_scrub else: result_set.add(ScheduledTaskController._execute_scrub_work.s(scrub_location=scrub_info[1], vdisk_guids=vdisk_guids_to_scrub).apply_async( routing_key='sr.{0}'.format(storage_router.machine_id) )) storage_router_list.append(storage_router) # Remote tasks have been launched, now start the local task and then wait for remote tasks to finish processed_guids = [] if local_scrub_location is not None and len(local_vdisks_to_scrub) > 0: try: processed_guids = ScheduledTaskController._execute_scrub_work(scrub_location=local_scrub_location, vdisk_guids=local_vdisks_to_scrub) except Exception as ex: logger.error('Gather Scrub - Storage Router {0:<15} - Scrubbing failed with error:\n - {1}'.format(local_storage_router.ip, ex)) all_results = result_set.join(propagate=False) # Propagate False makes sure all jobs are waited for even when 1 or more jobs fail for index, result in enumerate(all_results): if isinstance(result, list): processed_guids.extend(result) else: logger.error('Gather Scrub - Storage Router {0:<15} - Scrubbing failed with error:\n - {1}'.format(storage_router_list[index].ip, result)) if len(processed_guids) != len(vdisk_guids) or set(processed_guids).difference(vdisk_guids): raise RuntimeError('Scrubbing failed for 1 or more storagerouters') logger.info('Gather Scrub - Finished')
def test_result_set(self, manager): assert manager.inspect().ping() rs = ResultSet([add.delay(1, 1), add.delay(2, 2)]) assert rs.get(timeout=TIMEOUT) == [2, 4]
def get_check_prime(minimum, maximum): data = range(int(minimum), int(maximum)) chunks = [data[x:x+CHUNK_SIZE] for x in xrange(0, len(data), CHUNK_SIZE)] results = ResultSet(map(is_prime, chunks)) result = sum(results.get(), []) return json.dumps([x[0] for x in result if x[1]])