def prepare_celery(): """Runs the djcelery loader which is required to set the correct backend/scheduler Then adds some signal callbacks to ensure all tasks are registered correctly. """ djcelery.setup_loader() beat_init.connect(beat_update_schedule) worker_init.connect(celeryd_discover_tasks)
from base_task import TalentTask, _workerState from celery.signals import worker_init from genome_scoring import GENOME_FILE, TREE_FILE, BOUNDARY_FILE from sstree import PySSTree from blist import sortedlist import pickle def _initializeWorker(**kwargs): print '_initializeWorker called' organism = "drosophila_melanogaster" with open(GENOME_FILE % organism, 'r') as f: text = f.read() _workerState["sTree"] = PySSTree(text, "load", TREE_FILE % organism) with open(BOUNDARY_FILE % organism, "r") as f: _workerState["geneBoundaries"] = sortedlist(pickle.load(f), key=lambda x: x['pos']) worker_init.connect(_initializeWorker)
MausConfiguration from the Celery master process, the sub-process will always inherit the latest version of MausConfiguration from the master process. @param kwargs Arguments - unused. """ configuration = Configuration() config_doc = configuration.getConfigJSON() config_dictionary = json.loads(config_doc) MausConfiguration.version = config_dictionary["maus_version"] logger = logging.getLogger(__name__) if logger.isEnabledFor(logging.INFO): logger.info("MAUS version: %s" % MausConfiguration.version) # Bind the callback method to the Celery worker_init signal. worker_init.connect(worker_init_callback) def sub_process_broadcast(panel, func, arguments, errors): """ Invoke a function on every sub-process via asynchronous requests. This is based on a suggestion in http://groups.google.com/group/celery-developers/browse_thread/ thread/191389ab86185cce which repeatedly sends the same request to all the sub-processes until all confirm that they have executed the request. This is done via their returning of a status message and their sub-process ID. It is not the most efficient but it is the only way to ensure that all sub-processes are updated at the same time and remain consistent with both each other and the main process. @param panel Celery panel object.
headers = {"content-type": "application/json"} json_response = requests.post(fp_pred_url, data=payload, headers=headers) fingerprint_prediction = np.asarray(json.loads(json_response.text)['predictions']) fingerprint_prediction_nonzero = np.where(fingerprint_prediction.round()[0]==1)[0] # Tensorflow Serve fp mw query fp_pred_url = "http://smartfp-mw-tf-server:8501/v1/models/VGG16_high_aug_MW_continue:predict" payload = json.dumps({"instances": mat.reshape(1,200,240,1).tolist()}) headers = {"content-type": "application/json"} json_response = requests.post(fp_pred_url, data=payload, headers=headers) pred_MW = json.loads(json_response.text)['predictions'][0][0] print("FINISHED PREDICTION", time.time() - start_time, file=sys.stderr) DB = shared_model_data["DB"] topK = SMART_FPinder.search_database(fingerprint_prediction, fingerprint_prediction_nonzero, pred_MW, DB, mw=mw, top_candidates=20) topK.to_csv(output_result_table, index=None) print("FINISHED DB SEARCH", time.time() - start_time, file=sys.stderr) open(output_result_fp_pred, "w").write(json.dumps(fingerprint_prediction.tolist())) return 0 # Load the database when the worker starts worker_init.connect(worker_load_models)
rate_limit=rate_limit, run=handle_task, ignore_result=True, send_error_emails=False, __module__=create_task_queue.__module__)) def create_task_queues_from_yaml(app_root=None): if app_root: queue_info = _ParseQueueYaml(app_root) else: queue_info = None tasks = {} if queue_info and queue_info.queue: for entry in queue_info.queue: tasks[entry.name] = create_task_queue( entry.name, queueinfo.ParseRate(entry.rate), entry.bucket_size) else: tasks['default'] = create_task_queue('default', 5) return tasks def load_queue_config(signal, sender=None, **kwargs): if 'APP_ROOT' not in os.environ: logging.error("application root unknown") create_task_queues_from_yaml(os.environ.get('APP_ROOT')) worker_init.connect(load_queue_config)
class ExpensiveObject(): def __init__(self): self.id_ = ''.join( random.choice(string.ascii_lowercase) for _ in range(6)) print '>> initialising ExpensiveObject ' + self.id_ time.sleep(10) print '<< initialised ExpensiveObject ' + self.id_ def task(self, word): return '{} (from {})'.format(word.upper(), self.id_) def _load_expensive_object(**kwargs): if not _expensive_objects: _expensive_objects['loaded'] = ExpensiveObject() worker_init.connect(_load_expensive_object) @shared_task def expensive_task(word): logging.info('>> requesting `expensive_task` with ' + word) if 'loaded' not in _expensive_objects: raise Exception('Expensive object not yet loaded') expensive_object = _expensive_objects['loaded'] return expensive_object.task(word)