def run(self): task_family_name = str(self.task_family) if self.task_name == "ClarityNLPLuigiTask": self.task_name = task_family_name client = util.mongo_client() try: with self.output().open('w') as temp_file: temp_file.write("start writing custom task") jobs.update_job_status(str(self.job), util.conn_string, jobs.IN_PROGRESS, "Running Batch %s" % self.batch) self.pipeline_config = config.get_pipeline_config( self.pipeline, util.conn_string) jobs.update_job_status(str(self.job), util.conn_string, jobs.IN_PROGRESS, "Running Solr query") self.docs = solr_data.query( self.solr_query, rows=util.row_count, start=self.start, solr_url=util.solr_url, tags=self.pipeline_config.report_tags, mapper_inst=util.report_mapper_inst, mapper_url=util.report_mapper_url, mapper_key=util.report_mapper_key, types=self.pipeline_config.report_types, sources=self.pipeline_config.sources, filter_query=self.pipeline_config.filter_query, cohort_ids=self.pipeline_config.cohort, job_results_filters=self.pipeline_config.job_results) for d in self.docs: doc_id = d[util.solr_report_id_field] if util.use_memory_caching == "true": k = keys.hashkey(doc_id) document_cache[k] = d if util.use_redis_caching == "true": util.write_to_redis_cache("doc:" + doc_id, json.dumps(d)) jobs.update_job_status(str(self.job), util.conn_string, jobs.IN_PROGRESS, "Running %s main task" % self.task_name) self.run_custom_task(temp_file, client) temp_file.write("Done writing custom task!") self.docs = list() except Exception as ex: traceback.print_exc(file=sys.stderr) jobs.update_job_status(str(self.job), util.conn_string, jobs.WARNING, ''.join(traceback.format_stack())) print(ex) finally: client.close()
def get_race_data(document_id): if util.use_redis_caching == "true": util.add_cache_query_count() key = "raceFinder:" + str(document_id) res = util.get_from_redis_cache(key) if res: return json.loads(res) else: util.add_cache_compute_count() res2 = get_race_for_doc(document_id) util.write_to_redis_cache(key, json.dumps(res2)) return res2 elif util.use_memory_caching == "true": util.add_cache_query_count() return _get_race_data(document_id) else: return get_race_for_doc(document_id)
def get_race_data(document_id): if util.use_redis_caching == "true": util.add_cache_query_count() key = "raceFinder:" + str(document_id) res = util.get_from_redis_cache(key) if res: return json.loads(res) else: util.add_cache_compute_count() res2 = get_race_for_doc(document_id) util.write_to_redis_cache(key, json.dumps(res2)) return res2 elif util.use_memory_caching == "true": util.add_cache_query_count() return _get_race_data(document_id) else: return get_race_for_doc(document_id)
def get_document_by_id(document_id): doc = None if util.use_redis_caching == "true": util.add_cache_query_count() txt = util.get_from_redis_cache("doc:" + document_id) if not txt: util.add_cache_compute_count() doc = solr_data.query_doc_by_id(document_id, solr_url=util.solr_url) util.write_to_redis_cache("doc:" + document_id, json.dumps(doc)) else: doc = json.loads(txt) elif util.use_memory_caching == "true": util.add_cache_query_count() doc = _get_document_by_id(document_id) if not doc: return solr_data.query_doc_by_id(document_id, solr_url=util.solr_url) else: return doc
def get_job_performance(job_ids: list, connection_string: str): if not job_ids or len(job_ids) == 0: return dict() conn = psycopg2.connect(connection_string) cursor = conn.cursor() metrics = dict() try: in_clause = '' for i in job_ids: if len(in_clause) > 0: in_clause += ', ' in_clause += '%s' cursor.execute( """ SELECT status, nlp_job_id from nlp.nlp_job where nlp_job_id in ({}) """.format(in_clause), job_ids) status = None job_id = -1 statuses = cursor.fetchall() for s in statuses: status = s[0] job_id = s[1] metrics[job_id] = { "status": status, "final_results": 0, "final_subjects": 0, "intermediate_results": 0, "intermediate_subjects": 0, "counts_found": 0 } cursor.execute( """ SELECT status, description, date_updated, nlp_job_id from nlp.nlp_job_status where nlp_job_id in ({}) order by date_updated """.format(in_clause), job_ids) updates = cursor.fetchall() for row in updates: status_name = row[0] status_date = row[2] status_value = row[1] job_id = row[3] performance = metrics[job_id] counts_found = performance.get('counts_found', 0) if status_name == 'STATS_FINAL_SUBJECTS': performance['final_subjects'] = status_value counts_found += int(status_value) elif status_name == 'STATS_FINAL_RESULTS': performance['final_results'] = status_value counts_found += int(status_value) elif status_name == 'STATS_INTERMEDIATE_SUBJECTS': performance['intermediate_subjects'] = status_value elif status_name == 'STATS_INTERMEDIATE_RESULTS': performance['intermediate_results'] = status_value performance['counts_found'] = counts_found metrics[job_id] = performance for k in metrics.keys(): performance = metrics[k] counts_found = performance.get('counts_found', 0) if counts_found == 0 and status == COMPLETED: final_subjects = util.get_from_redis_cache( 'final_subjects_{}'.format(k)) final_results = util.get_from_redis_cache( 'final_results{}'.format(k)) if final_results and final_subjects: performance['final_subjects'] = final_subjects performance['final_results'] = final_results else: stats = phenotype_stats(str(job_id), True) performance['final_subjects'] = stats["subjects"] performance['final_results'] = stats["results"] util.write_to_redis_cache('final_subjects_{}'.format(k), stats["subjects"]) util.write_to_redis_cache('final_results_{}'.format(k), stats["results"]) int_subjects = util.get_from_redis_cache( 'intermediate_subjects_{}'.format(k)) int_results = util.get_from_redis_cache( 'intermediate_results{}'.format(k)) if int_subjects and int_results: performance['intermediate_subjects'] = int_subjects performance['intermediate_results'] = int_results else: intermediate_stats = phenotype_stats(str(job_id), False) performance['intermediate_subjects'] = intermediate_stats[ "subjects"] performance['intermediate_results'] = intermediate_stats[ "results"] util.write_to_redis_cache( 'intermediate_subjects_{}'.format(k), intermediate_stats["subjects"]) util.write_to_redis_cache( 'intermediate_results_{}'.format(k), intermediate_stats["results"]) if 'counts_found' in performance: del performance['counts_found'] metrics[job_id] = performance except Exception as ex: traceback.print_exc(file=sys.stdout) finally: conn.close() return metrics