def diagnose(self, content, diseases_only=False, content_date=None, use_infection_annotator=False, include_incidents=False): time_sofar = time_sofar_gen(datetime.datetime.now()) base_keyword_dict = self.keyword_extractor.transform([content])[0] feature_dict = self.keyword_processor.transform([base_keyword_dict]) X = self.dict_vectorizer.transform(feature_dict)[0] logger.info(time_sofar.next() + 'Computed feature vector') def diagnosis(i, p): scores = self.classifier.coef_[i] * X # Scores are normalized so they can be compared across different # classifications. norm = np.linalg.norm(scores) if norm > 0: scores /= norm scores *= p # These might be numpy types. I coerce them to native python # types so we can easily serialize the output as json. scored_keywords = zip(self.keywords, scores) keyword_scores = {} for keyword, score in scored_keywords: if score > 0 and keyword in base_keyword_dict: keyword_scores[keyword] = float(score) return { 'name': unicode(self.classifier.classes_[i]), 'probability': float(p), 'keywords': [{ 'name': unicode(kwd), 'score': float(score), } for kwd, score in scored_keywords if score > 0 and kwd in base_keyword_dict], 'inferred_keywords': [{ 'name': unicode(kwd), 'score': float(score), } for kwd, score in scored_keywords if score > 0 and kwd not in base_keyword_dict] } diseases = [diagnosis(i, p) for i, p in self.best_guess(X)] if diseases_only: return {'diseases': diseases} logger.info(time_sofar.next() + 'Diagnosed diseases') anno_doc = AnnoDoc(content, date=content_date) anno_doc.add_tier(self.keyword_annotator) logger.info('keywords annotated') anno_doc.add_tier(self.resolved_keyword_annotator) logger.info('resolved keywords annotated') anno_doc.add_tier(self.date_annotator) logger.info('dates annotated') if use_infection_annotator: anno_doc.add_tier(self.infection_annotator) anno_doc.tiers['counts'] = anno_doc.tiers.pop('infections') attribute_remappings = {'infection': 'case'} for span in anno_doc.tiers['counts']: span.metadata['attributes'] = [ attribute_remappings.get(attribute, attribute) for attribute in span.metadata['attributes'] ] else: anno_doc.add_tier(self.count_annotator) logger.info('counts annotated') anno_doc.add_tier(self.geoname_annotator) logger.info('geonames annotated') anno_doc.add_tier(StructuredIncidentAnnotator()) logger.info('structured incidents annotated') anno_doc.filter_overlapping_spans(tier_names=[ 'dates', 'geonames', 'diseases', 'hosts', 'modes', 'pathogens', 'symptoms' ]) logger.info('filtering overlapping spans done') dates = [] for span in anno_doc.tiers['dates']: range_start, range_end = span.datetime_range dates.append({ 'type': 'datetime', 'name': span.text, 'value': span.text, 'textOffsets': [[span.start, span.end]], 'timeRange': { 'beginISO': range_start.isoformat().split('T')[0], 'begin': { 'year': range_start.year, 'month': range_start.month, 'date': range_start.day }, # The date range does not include the end day. 'endISO': range_end.isoformat().split('T')[0], 'end': { 'year': range_end.year, 'month': range_end.month, 'date': range_end.day }, } }) geonames_grouped = {} for span in anno_doc.tiers['geonames']: if not span.geoname['geonameid'] in geonames_grouped: geonames_grouped[span.geoname['geonameid']] = { 'type': 'location', 'name': span.geoname.name, 'geoname': span.geoname.to_dict(), 'textOffsets': [[span.start, span.end]] } else: geonames_grouped[ span.geoname['geonameid']]['textOffsets'].append( [span.start, span.end]) logger.info(time_sofar.next() + 'Annotated geonames') counts = [] for span in anno_doc.tiers['counts'].without_overlaps( anno_doc.tiers['structured_data']): count_dict = dict(span.metadata) count_dict['type'] = 'count' count_dict['text'] = span.text count_dict['label'] = span.label count_dict['textOffsets'] = [[span.start, span.end]] counts.append(count_dict) # Include legacy case counts so the diagnositic dashboard # doesn't break. if 'case' in count_dict['attributes']: counts.append({ 'type': 'caseCount', 'text': count_dict['text'], 'value': count_dict['count'], 'modifiers': count_dict['attributes'], 'cumulative': "cumulative" in count_dict['attributes'], 'textOffsets': count_dict['textOffsets'] }) keyword_types = ['diseases', 'hosts', 'modes', 'pathogens', 'symptoms'] keyword_groups = {} for keyword_type in keyword_types: keyword_groups[keyword_type] = {} for span in anno_doc.tiers['keyword.' + keyword_type]: if span.label not in keyword_groups[keyword_type]: keyword_groups[keyword_type][span.label] = { 'type': keyword_type, 'value': span.label, 'textOffsets': [[span.start, span.end]] } else: keyword_groups[keyword_type][ span.label]['textOffsets'].append( [span.start, span.end]) resolved_keywords = [] for span in anno_doc.tiers['resolved_keywords'].without_overlaps( anno_doc.tiers['geonames']): resolved_keywords.append({ 'type': 'resolvedKeyword', 'resolutions': span.metadata['resolutions'], 'text': span.text, 'textOffsets': [[span.start, span.end]] }) result = { 'diagnoserVersion': self.__version__, 'dateOfDiagnosis': datetime.datetime.now(), 'diseases': diseases, 'structuredIncidents': [ dict(span.metadata, textOffsets=[[span.start, span.end]]) for span in anno_doc.tiers['structured_incidents']], 'features': counts +\ geonames_grouped.values() +\ dates +\ keyword_groups['diseases'].values() +\ keyword_groups['hosts'].values() +\ keyword_groups['modes'].values() +\ keyword_groups['pathogens'].values() +\ keyword_groups['symptoms'].values() +\ resolved_keywords} if include_incidents: result['incidents'] = [] anno_doc.add_tier(IncidentAnnotator()) for incident_span in anno_doc.tiers['incidents']: metadata = incident_span.metadata incident_data = { 'offsets': [span.start, span.end], 'type': metadata['type'], 'value': metadata['value'], 'dateRange': [ d.isoformat().split('T')[0] for d in metadata['dateRange'] ], 'locations': metadata['locations'], 'species': metadata['species'], 'status': metadata.get('status'), 'resolvedDisease': metadata.get('resolvedDisease'), 'annotations': { 'case': [{ 'offsets': [incident_span.start, incident_span.end] }] } } if 'count_annotation' in metadata: count_annotation = metadata['count_annotation'] incident_data['annotations'] = { 'case': [{ 'offsets': [count_annotation.start, count_annotation.end] }], 'date': [{ 'offsets': [anno.start, anno.end] } for anno in metadata['date_territory'].metadata], 'location': [{ 'offsets': [anno.start, anno.end] } for anno in metadata['geoname_territory'].metadata], 'disease': [{ 'offsets': [anno.start, anno.end] } for anno in metadata['disease_territory'].metadata] } result['incidents'].append(incident_data) return result
def diagnose(self, content, diseases_only=False, content_date=None): time_sofar = time_sofar_gen(datetime.datetime.now()) base_keyword_dict = self.keyword_extractor.transform([content])[0] feature_dict = self.keyword_processor.transform([base_keyword_dict]) X = self.dict_vectorizer.transform(feature_dict)[0] logger.info(time_sofar.next() + 'Computed feature vector') def diagnosis(i, p): scores = self.classifier.coef_[i] * X # Scores are normalized so they can be compared across different # classifications. norm = np.linalg.norm(scores) if norm > 0: scores /= norm scores *= p # These might be numpy types. I coerce them to native python # types so we can easily serialize the output as json. scored_keywords = zip(self.keywords, scores) keyword_scores = {} for keyword, score in scored_keywords: if score > 0 and keyword in base_keyword_dict: keyword_scores[keyword] = float(score) return { 'name': unicode(self.classifier.classes_[i]), 'probability': float(p), 'keywords': [{ 'name': unicode(kwd), 'score': float(score), } for kwd, score in scored_keywords if score > 0 and kwd in base_keyword_dict], 'inferred_keywords': [{ 'name': unicode(kwd), 'score': float(score), } for kwd, score in scored_keywords if score > 0 and kwd not in base_keyword_dict] } diseases = [diagnosis(i, p) for i, p in self.best_guess(X)] if diseases_only: return {'diseases': diseases} logger.info(time_sofar.next() + 'Diagnosed diseases') anno_doc = AnnoDoc(content, date=content_date) anno_doc.add_tier(self.keyword_annotator) logger.info('keywords annotated') anno_doc.add_tier(self.resolved_keyword_annotator) logger.info('resolved keywords annotated') anno_doc.add_tier(self.date_annotator) logger.info('dates annotated') anno_doc.add_tier(self.count_annotator) logger.info('counts annotated') anno_doc.add_tier(self.geoname_annotator) logger.info('geonames annotated') anno_doc.filter_overlapping_spans(tier_names=[ 'dates', 'geonames', 'diseases', 'hosts', 'modes', 'pathogens', 'symptoms' ]) logger.info('filtering overlapping spans done') dates = [] for span in anno_doc.tiers['dates'].spans: range_start, range_end = span.datetime_range dates.append({ 'type': 'datetime', 'name': span.label, 'value': span.label, 'textOffsets': [[span.start, span.end]], 'timeRange': { 'begin': { 'year': range_start.year, 'month': range_start.month, 'date': range_start.day }, 'end': { 'year': range_end.year, 'month': range_end.month, 'date': range_end.day }, } }) geonames_grouped = {} for span in anno_doc.tiers['geonames'].spans: if not span.geoname['geonameid'] in geonames_grouped: geonames_grouped[span.geoname['geonameid']] = { 'type': 'location', 'name': span.label, 'geoname': span.geoname.to_dict(), 'textOffsets': [[span.start, span.end]] } else: geonames_grouped[ span.geoname['geonameid']]['textOffsets'].append( [span.start, span.end]) logger.info(time_sofar.next() + 'Annotated geonames') counts = [] for span in anno_doc.tiers['counts'].spans: count_dict = span.to_dict() count_dict['type'] = 'count' counts.append(count_dict) # Include legacy case counts so the diagnositic dashboard # doesn't break. if "case" in count_dict['attributes']: counts.append({ 'type': "caseCount", 'text': count_dict['text'], 'value': count_dict['label'], 'modifiers': count_dict['attributes'], 'cumulative': "cumulative" in count_dict['attributes'], 'textOffsets': count_dict['textOffsets'] }) keyword_types = ['diseases', 'hosts', 'modes', 'pathogens', 'symptoms'] keyword_groups = {} for keyword_type in keyword_types: keyword_groups[keyword_type] = {} for span in anno_doc.tiers[keyword_type].spans: if not span.label in keyword_groups[keyword_type]: keyword_groups[keyword_type][span.label] = { 'type': keyword_type, 'value': span.label, 'textOffsets': [[span.start, span.end]] } else: keyword_groups[keyword_type][ span.label]['textOffsets'].append( [span.start, span.end]) resolved_keywords = [] for span in anno_doc.tiers['resolved_keywords'].spans: resolved_keywords.append({ 'type': 'resolvedKeyword', 'resolutions': span.resolutions, 'text': span.text, 'textOffsets': [[span.start, span.end]] }) return { 'diagnoserVersion': self.__version__, 'dateOfDiagnosis': datetime.datetime.now(), 'diseases': diseases, 'features': counts +\ geonames_grouped.values() +\ dates +\ keyword_groups['diseases'].values() +\ keyword_groups['hosts'].values() +\ keyword_groups['modes'].values() +\ keyword_groups['pathogens'].values() +\ keyword_groups['symptoms'].values() +\ resolved_keywords}
def diagnose( self, content, diseases_only=False, content_date=None, use_infection_annotator=False, include_incidents=False): time_sofar = time_sofar_gen(datetime.datetime.now()) base_keyword_dict = self.keyword_extractor.transform([content])[0] feature_dict = self.keyword_processor.transform([base_keyword_dict]) X = self.dict_vectorizer.transform(feature_dict)[0] logger.info(time_sofar.next() + 'Computed feature vector') def diagnosis(i, p): scores = self.classifier.coef_[i] * X # Scores are normalized so they can be compared across different # classifications. norm = np.linalg.norm(scores) if norm > 0: scores /= norm scores *= p # These might be numpy types. I coerce them to native python # types so we can easily serialize the output as json. scored_keywords = zip(self.keywords, scores) keyword_scores = {} for keyword, score in scored_keywords: if score > 0 and keyword in base_keyword_dict: keyword_scores[keyword] = float(score) return { 'name': unicode(self.classifier.classes_[i]), 'probability': float(p), 'keywords': [{ 'name': unicode(kwd), 'score': float(score), } for kwd, score in scored_keywords if score > 0 and kwd in base_keyword_dict], 'inferred_keywords': [{ 'name': unicode(kwd), 'score': float(score), } for kwd, score in scored_keywords if score > 0 and kwd not in base_keyword_dict] } diseases = [diagnosis(i,p) for i,p in self.best_guess(X)] if diseases_only: return { 'diseases': diseases } logger.info(time_sofar.next() + 'Diagnosed diseases') anno_doc = AnnoDoc(content, date=content_date) anno_doc.add_tier(self.keyword_annotator) logger.info('keywords annotated') anno_doc.add_tier(self.resolved_keyword_annotator) logger.info('resolved keywords annotated') anno_doc.add_tier(self.date_annotator) logger.info('dates annotated') if use_infection_annotator: anno_doc.add_tier(self.infection_annotator) anno_doc.tiers['counts'] = anno_doc.tiers.pop('infections') attribute_remappings = { 'infection': 'case' } for span in anno_doc.tiers['counts']: span.metadata['attributes'] = [ attribute_remappings.get(attribute, attribute) for attribute in span.metadata['attributes']] else: anno_doc.add_tier(self.count_annotator) logger.info('counts annotated') anno_doc.add_tier(self.geoname_annotator) logger.info('geonames annotated') anno_doc.add_tier(StructuredIncidentAnnotator()) logger.info('structured incidents annotated') anno_doc.filter_overlapping_spans( tier_names=[ 'dates', 'geonames', 'diseases', 'hosts', 'modes', 'pathogens', 'symptoms' ] ) logger.info('filtering overlapping spans done') dates = [] for span in anno_doc.tiers['dates']: range_start, range_end = span.datetime_range dates.append({ 'type': 'datetime', 'name': span.text, 'value': span.text, 'textOffsets': [ [span.start, span.end] ], 'timeRange': { 'beginISO': range_start.isoformat().split('T')[0], 'begin': { 'year': range_start.year, 'month': range_start.month, 'date': range_start.day }, # The date range does not include the end day. 'endISO': range_end.isoformat().split('T')[0], 'end': { 'year': range_end.year, 'month': range_end.month, 'date': range_end.day }, } }) geonames_grouped = {} for span in anno_doc.tiers['geonames']: if not span.geoname['geonameid'] in geonames_grouped: geonames_grouped[span.geoname['geonameid']] = { 'type': 'location', 'name': span.geoname.name, 'geoname': span.geoname.to_dict(), 'textOffsets': [ [span.start, span.end] ] } else: geonames_grouped[ span.geoname['geonameid'] ]['textOffsets'].append( [span.start, span.end] ) logger.info(time_sofar.next() + 'Annotated geonames') counts = [] for span in anno_doc.tiers['counts'].without_overlaps(anno_doc.tiers['structured_data']): count_dict = dict(span.metadata) count_dict['type'] = 'count' count_dict['text'] = span.text count_dict['label']= span.label count_dict['textOffsets']= [[span.start, span.end]] counts.append(count_dict) # Include legacy case counts so the diagnositic dashboard # doesn't break. if 'case' in count_dict['attributes']: counts.append({ 'type': 'caseCount', 'text': count_dict['text'], 'value': count_dict['count'], 'modifiers': count_dict['attributes'], 'cumulative': "cumulative" in count_dict['attributes'], 'textOffsets': count_dict['textOffsets'] }) keyword_types = ['diseases', 'hosts', 'modes', 'pathogens', 'symptoms'] keyword_groups = {} for keyword_type in keyword_types: keyword_groups[keyword_type] = {} for span in anno_doc.tiers['keyword.' + keyword_type]: if span.label not in keyword_groups[keyword_type]: keyword_groups[keyword_type][span.label] = { 'type': keyword_type, 'value': span.label, 'textOffsets': [[span.start, span.end]] } else: keyword_groups[keyword_type][span.label]['textOffsets'].append( [span.start, span.end] ) resolved_keywords = [] for span in anno_doc.tiers['resolved_keywords'].without_overlaps(anno_doc.tiers['geonames']): resolved_keywords.append({ 'type': 'resolvedKeyword', 'resolutions': span.metadata['resolutions'], 'text': span.text, 'textOffsets': [[span.start, span.end]]}) result = { 'diagnoserVersion': self.__version__, 'dateOfDiagnosis': datetime.datetime.now(), 'diseases': diseases, 'structuredIncidents': [ dict(span.metadata, textOffsets=[[span.start, span.end]]) for span in anno_doc.tiers['structured_incidents']], 'features': counts +\ geonames_grouped.values() +\ dates +\ keyword_groups['diseases'].values() +\ keyword_groups['hosts'].values() +\ keyword_groups['modes'].values() +\ keyword_groups['pathogens'].values() +\ keyword_groups['symptoms'].values() +\ resolved_keywords} if include_incidents: result['incidents'] = [] anno_doc.add_tier(IncidentAnnotator()) for incident_span in anno_doc.tiers['incidents']: metadata = incident_span.metadata incident_data = { 'offsets': [span.start, span.end], 'type': metadata['type'], 'value': metadata['value'], 'dateRange': [d.isoformat().split('T')[0] for d in metadata['dateRange']], 'locations': metadata['locations'], 'species': metadata['species'], 'status': metadata.get('status'), 'resolvedDisease': metadata.get('resolvedDisease'), 'annotations': { 'case': [{ 'offsets': [incident_span.start, incident_span.end] }] } } if 'count_annotation' in metadata: count_annotation = metadata['count_annotation'] incident_data['annotations'] = { 'case': [{ 'offsets': [count_annotation.start, count_annotation.end] }], 'date': [ { 'offsets': [anno.start, anno.end] } for anno in metadata['date_territory'].metadata ], 'location': [ { 'offsets': [anno.start, anno.end] } for anno in metadata['geoname_territory'].metadata ], 'disease': [ { 'offsets': [anno.start, anno.end] } for anno in metadata['disease_territory'].metadata ] } result['incidents'].append(incident_data) return result