def configure_extensions(self): Empty.configure_extensions(self) self.celery = Celery(self.name, broker=self.config['CELERY_BROKER_URL'], beat=True) self.celery.conf.update(self.config) app = self self.redis = self.celery.broker_connection().default_channel.client if 'root_path' in self.config: self.root_path = self.config['root_path'] if 'WHYIS_TEMPLATE_DIR' in self.config and app.config['WHYIS_TEMPLATE_DIR'] is not None: my_loader = jinja2.ChoiceLoader( [jinja2.FileSystemLoader(p) for p in self.config['WHYIS_TEMPLATE_DIR']] + [app.jinja_loader] ) app.jinja_loader = my_loader def setup_task(service): service.app = app print(service) result = None if service.query_predicate == self.NS.whyis.globalChangeQuery: result = process_resource else: result = process_nanopub result.service = lambda : service return result @self.celery.task def process_resource(service_name, taskid=None): service = self.config['inferencers'][service_name] if is_waiting(service_name): print("Deferring to a later invocation.", service_name) return print(service_name) service.process_graph(app.db) @self.celery.task def process_nanopub(nanopub_uri, service_name, taskid=None): service = self.config['inferencers'][service_name] print(service, nanopub_uri) if app.nanopub_manager.is_current(nanopub_uri): nanopub = app.nanopub_manager.get(nanopub_uri) service.process_graph(nanopub) else: print("Skipping retired nanopub", nanopub_uri) def setup_periodic_task(task): @self.celery.task def find_instances(): print("Triggered task", task['name']) for x, in task['service'].getInstances(app.db): task['do'](x) @self.celery.task def do_task(uri): print("Running task", task['name'], 'on', uri) resource = app.get_resource(uri) # result never used task['service'].process_graph(resource.graph) task['service'].app = app task['find_instances'] = find_instances task['do'] = do_task return task app.inference_tasks = [] if 'inference_tasks' in self.config: app.inference_tasks = [setup_periodic_task(task) for task in self.config['inference_tasks']] for name, task in list(self.config['inferencers'].items()): task.app = app for task in app.inference_tasks: if 'schedule' in task: #print "Scheduling task", task['name'], task['schedule'] self.celery.add_periodic_task( crontab(**task['schedule']), task['find_instances'].s(), name=task['name'] ) else: task['find_instances'].delay() def is_waiting(service_name): """ Check if a task is waiting. """ scheduled_tasks = list(inspect().scheduled().values())[0] for task in scheduled_tasks: if 'kwargs' in task: args = eval(task['kwargs']) if service_name == args.get('service_name',None): return True return False def is_running_waiting(service_name): """ Check if a task is running or waiting. """ if is_waiting(service_name): return True running_tasks = list(inspect().active().values())[0] for task in running_tasks: if 'kwargs' in task: args = eval(task['kwargs']) if service_name == args.get('service_name',None): return True return False @self.celery.task() def update(nanopub_uri): '''gets called whenever there is a change in the knowledge graph. Performs a breadth-first knowledge expansion of the current change.''' #print "Updating on", nanopub_uri if not app.nanopub_manager.is_current(nanopub_uri): print("Skipping retired nanopub", nanopub_uri) return nanopub = app.nanopub_manager.get(nanopub_uri) nanopub_graph = ConjunctiveGraph(nanopub.store) if 'inferencers' in self.config: for name, service in list(self.config['inferencers'].items()): service.app = self if service.query_predicate == self.NS.whyis.updateChangeQuery: #print "checking", name, nanopub_uri, service.get_query() if service.getInstances(nanopub_graph): print("invoking", name, nanopub_uri) process_nanopub.apply_async(kwargs={'nanopub_uri': nanopub_uri, 'service_name':name}, priority=1 ) for name, service in list(self.config['inferencers'].items()): service.app = self if service.query_predicate == self.NS.whyis.globalChangeQuery and not is_running_waiting(name): #print "checking", name, service.get_query() process_resource.apply_async(kwargs={'service_name':name}, priority=5) def run_update(nanopub_uri): update.apply_async(args=[nanopub_uri],priority=9) self.nanopub_update_listener = run_update def is_waiting_importer(entity_name, exclude=None): """ Check if a task is running or waiting. """ if inspect().scheduled(): tasks = list(inspect().scheduled().values()) for task in tasks: if 'args' in task and entity_name in task['args']: return True return False app = self @self.celery.task(retry_backoff=True, retry_jitter=True,autoretry_for=(Exception,),max_retries=4, bind=True) def run_importer(self, entity_name): entity_name = URIRef(entity_name) counter = app.redis.incr(("import",entity_name)) if counter > 1: return print('importing', entity_name) importer = app.find_importer(entity_name) if importer is None: return importer.app = app modified = importer.last_modified(entity_name, app.db, app.nanopub_manager) updated = importer.modified(entity_name) if updated is None: updated = datetime.now(pytz.utc) print("Remote modified:", updated, type(updated), "Local modified:", modified, type(modified)) if modified is None or (updated - modified).total_seconds() > importer.min_modified: importer.load(entity_name, app.db, app.nanopub_manager) app.redis.set(("import",entity_name),0) self.run_importer = run_importer self.template_imports = {} if 'template_imports' in self.config: for name, imp in list(self.config['template_imports'].items()): try: m = importlib.import_module(imp) self.template_imports[name] = m except Exception: print("Error importing module %s into template variable %s." % (imp, name)) raise
def configure_extensions(self): Empty.configure_extensions(self) self.celery = Celery(self.name, broker=self.config['CELERY_BROKER_URL'], beat=True) self.celery.conf.update(self.config) app = self if 'root_path' in self.config: self.root_path = self.config['root_path'] if 'SATORU_TEMPLATE_DIR' in self.config and app.config[ 'SATORU_TEMPLATE_DIR'] is not None: my_loader = jinja2.ChoiceLoader([ jinja2.FileSystemLoader(p) for p in self.config['SATORU_TEMPLATE_DIR'] ] + [app.jinja_loader]) app.jinja_loader = my_loader def setup_task(service): service.app = app print service result = None if service.query_predicate == self.NS.graphene.globalChangeQuery: result = process_resource else: result = process_nanopub result.service = lambda: service return result @self.celery.task def process_resource(service_name): service = self.config['inferencers'][service_name] print service service.process_graph(app.db) @self.celery.task def process_nanopub(nanopub_uri, service_name): service = self.config['inferencers'][service_name] print service, nanopub_uri nanopub = app.nanopub_manager.get(nanopub_uri) service.process_graph(nanopub) def setup_periodic_task(task): @self.celery.task def find_instances(): print "Triggered task", task['name'] for x, in app.db.query(task['service'].get_query()): task['do'](x) @self.celery.task def do_task(uri): print "Running task", task['name'], 'on', uri resource = app.get_resource(uri) result = task['service'].process_graph(resource.graph) task['service'].app = app task['find_instances'] = find_instances task['do'] = do_task return task app.inference_tasks = [] if 'inference_tasks' in self.config: app.inference_tasks = [ setup_periodic_task(task) for task in self.config['inference_tasks'] ] for task in app.inference_tasks: if 'schedule' in task: #print "Scheduling task", task['name'], task['schedule'] self.celery.add_periodic_task(crontab(**task['schedule']), task['find_instances'].s(), name=task['name']) else: task['find_instances'].delay() @self.celery.task() def update(nanopub_uri): '''gets called whenever there is a change in the knowledge graph. Performs a breadth-first knowledge expansion of the current change.''' #print "Updating on", nanopub_uri nanopub = app.nanopub_manager.get(nanopub_uri) nanopub_graph = ConjunctiveGraph(nanopub.store) if 'inferencers' in self.config: for name, service in self.config['inferencers'].items(): service.app = self if service.query_predicate == self.NS.graphene.globalChangeQuery: #print "checking", name, service.get_query() process_resource(name) if service.query_predicate == self.NS.graphene.updateChangeQuery: #print "checking", name, nanopub_uri, service.get_query() if len(list(nanopub_graph.query( service.get_query()))) > 0: print "invoking", name, nanopub_uri process_nanopub(nanopub_uri, name) def run_update(nanopub_uri): update.delay(nanopub_uri) self.nanopub_update_listener = run_update @self.celery.task(retry_backoff=True, retry_jitter=True, autoretry_for=(Exception, ), max_retries=4) def run_importer(entity_name): importer = self.find_importer(entity_name) modified = importer.last_modified(entity_name, self.db, self.nanopub_manager) updated = importer.modified(entity_name) if updated is None: updated = datetime.now(pytz.utc) print "Remote modified:", updated, type( updated), "Local modified:", modified, type(modified) if modified is None or (updated - modified ).total_seconds() > importer.min_modified: importer.load(entity_name, self.db, self.nanopub_manager) self.run_importer = run_importer