def prepareGloveModel():
ConfigReader('../conf/smartanno_conf.json')
glove_path = ConfigReader.getValue('glove/model_path')
glove_vocab = ConfigReader.getValue('glove/vocab')
glove_vector = ConfigReader.getValue('glove/vector')
GloveModel(word2vec_file=glove_path, vocab=glove_vocab, vect=glove_vector)
gm = GloveModel.glove_model
def start(self):
if ConfigReader.getValue("glove/model_path") is None or len(ConfigReader.getValue("glove/model_path")) == 0:
self.workflow.steps[self.pos_id + 2].start()
return
if not hasattr(self.workflow, 'we_extended'):
self.workflow.we_extended = dict()
rows = self.showWords(self.workflow.filters)
self.box = widgets.VBox(rows, layout=widgets.Layout(display='flex', flex_grown='column'))
display(self.box)
pass
def start(self):
cr = ConfigReader()
self.workflow = Workflow([
IntroStep(
'<h2>Welcome to SmartAnno!</h2><h4>Do you want to start from beginning or continue previous reviewing? </h4>',
name='intro'),
DBInitiater(name='db_initiator'),
DirChooser(name='choosedir'),
ReadFiles(name='readfiles'),
DocsToDB(name='save2db'),
TaskChooser(name='tasknamer'),
DataSetChooser(
name='dataset_chooser',
description='<h4>Choose which dateaset you want to use: </h4>'
),
AnnotationTypeDef(
'<h3>Annotation types:</h3><p>List all the types you want to identify below. Each type per line.<br/>If you'
'have too many types, try set up them separately, so that you won't need to choose from a long list '
'for each sample. </p>',
name='types'),
KeywordsFiltering(name='keywords'),
# PreviousNextIntSlider(value=60, min=0, max=100, step=10,
# description='<h4>Percentage to Filter: </h4><p>Choose how many percent of the samples '
# 'you want to use the keywords filter.</p>', name='percent2filter'),
KeywordsUMLSExtenderSetup(name='umls_extender_setup'),
KeywordsUMLSExtender(
name='umls_extender',
sources=cr.getValue("umls/sources"),
filter_by_length=cr.getValue("umls/filter_by_length"),
filter_by_contains=cr.getValue("umls/filter_by_contains"),
max_query=cr.getValue("umls/max_query")),
KeywordsEmbeddingExtenderSetup(name='w_e_extender_setup'),
KeywordsEmbeddingExtender(name='w_e_extender', max_query=40),
ReviewRBInit(name="rb_review_init"),
ReviewRBLoop(name='rb_review'),
PreviousNextHTML(
description=
'<h2>Congratuations!</h2><h4>You have finished the initial review '
'on the rule-base preannotations. </h4>',
name='rb_review_done'),
ReviewMLInit(name='ml_review_init'),
ReviewMLLoop(name='ml_review',
ml_classifier_cls=self.ml_classifier_cls),
PreviousNextHTML(
name='finish',
description=
'<h3>Well done!</h3><h4>Now you have finished reviewing all the samples. '
)
])
self.workflow.start(False)
pass
def __init__(self,
targets=None,
modifiers=None,
feature_inference_rule=None,
document_inference_rule=None,
rush_rule=None,
expected_values=[],
save_markups=True):
self.document_inferencer = DocumentInferencer(document_inference_rule)
self.feature_inferencer = FeatureInferencer(feature_inference_rule)
self.conclusions = []
self.modifiers = modifiers
self.targets = targets
self.save_markups = save_markups
self.expected_values = [value.lower() for value in expected_values]
self.saved_markups_map = dict()
self.pyrush = None
if rush_rule is None or not os.path.isfile(rush_rule):
rush_rule = ConfigReader.getValue('rush_rules_path')
if rush_rule is not None and os.path.isfile(rush_rule):
self.pyrush = RuSH(rush_rule)
else:
logMsg(("File not found", os.path.abspath(rush_rule)))
self.last_doc_name = ''
if modifiers is not None and targets is not None:
if isinstance(modifiers, str) and isinstance(targets, str):
if (modifiers.endswith('.csv') or modifiers.endswith('.tsv') or modifiers.endswith(
'.txt') or modifiers.endswith('.yml')) \
and (targets.endswith('.csv') or targets.endswith('.tsv') or targets.endswith(
'.txt') or targets.endswith('.yml') or targets.startswith('Lex\t')):
self.setModifiersTargetsFromFiles(modifiers, targets)
else:
self.setModifiersTargets(modifiers, targets)
RBDocumentClassifier.instance = self
def __init__(self, name=None):
super().__init__(name)
self.dao = None
self.dbpath = ''
self.remove_old = False
self.dataset_name = 'orig'
self.whoosh_root = ConfigReader.getValue("whoosh/root_path")
self.html1 = widgets.HTML('<h4>Give a name for this dataset: </h4>')
self.dataset_name_input = None
self.html2 = None
self.toggle = widgets.ToggleButtons(
options=['TextBlob_Splitter', 'PyRuSh', 'Not_To_Split'],
description='',
disabled=False,
value='Not_To_Split',
button_style='', # 'success', 'info', 'warning', 'danger' or ''
tooltips=[
'Use TextBlob sentence splitter',
'Use PyRuSH to split sentences', 'don\'t split'
],
layout=widgets.Layout(width='70%')
# icons=['check'] * 3
)
self.data_step = None
pass
def __init__(self,
description='',
name=str(Step.global_id + 1),
ml_classifier_cls=LogisticBOWClassifier):
self.sample_size_input = None
self.percent_slider = None
self.samples = {"contain": [], "notcontain": []}
self.box = None
self.data = None
self.docs = None
self.annos = None
self.reviewed_docs = None
self.reviewed_pos = None
self.leftover = None
self.ready = False
# reset, continue, addmore,
self.move_next_option = ''
self.previousReviewed = OrderedDict()
self.learning_pace = ConfigReader.getValue('review/ml_learning_pace')
self.un_reviewed = 0
self.parameters = dict()
self.parameter_inputs = dict()
self.ml_classifier_cls = ml_classifier_cls
super().__init__(name=name)
pass
def __init__(self, apikey=None):
# self.username=username
# self.password=password
if apikey is not None:
self.apikey = apikey
else:
self.apikey = ConfigReader.getValue('api_key')
self.service = "http://umlsks.nlm.nih.gov"
def __init__(self, description='', name=None):
self.glove_path = ConfigReader.getValue('glove/model_path')
self.glove_vocab = ConfigReader.getValue('glove/vocab')
self.glove_vector = ConfigReader.getValue('glove/vector')
# widgets to take the user inputs
self.glove_path_input = None
self.glove_vocab_input = None
self.glove_vector_input = None
self.api_key_input = None
if self.glove_vocab is None:
self.glove_vocab = 1900000
if self.glove_vector is None:
self.glove_vector = 300
self.html = widgets.HTML(value=description)
super().__init__(name)
pass
def __init__(
self,
description='<h4>Extend keywords through <b>UMLS</b></h4><p>Please select which keywords you want to '
'check the synonyms from UMLS:',
name=None):
self.api_key = ConfigReader.getValue('api_key')
self.title = widgets.HTML(value=description)
self.to_ext_words = dict()
self.to_umls_ext_filters = dict()
self.api_input = None
super().__init__(name)
def __init__(self, name=str(Step.global_id + 1), **kwargs):
super().__init__([], name=name)
self.docs = []
self.data = dict()
self.annos = dict()
self.reviewed_docs = dict()
self.threshold = ConfigReader.getValue('review/rb_model_threshold')
self.nlp = None
self.js = '''<script>
function setFocusToTextBox(){
var spans = document.getElementsByClassName("highlighter");
var id=document.getElementById('d1').pos
if (id===undefined){
id=0
}
if (id>=spans.length){
id=0
}
var topPos = spans[id].offsetTop;
dv=document.getElementById('d1')
dv.scrollTop = topPos-20;
dv.pos=id+1;
}
</script>'''
self.end_js = '''<script>document.getElementById('d1').pos=0;topPos=0;</script>'''
self.matcher = None
self.metaColumns = ConfigReader().getValue("review/meta_columns")
self.div_height = ConfigReader().getValue("review/div_height")
logMsg(('self.div_height:', self.div_height))
self.show_meta_name = ConfigReader().getValue("review/show_meta_name")
self.hightligh_span_tag = ' <span class="highlighter" style="background-color: %s ">' % ConfigReader(
).getValue("review/highlight_color")
if 'rush_rule' in kwargs:
self.rush_rule = kwargs['rush_rule']
else:
self.rush_rule = ConfigReader.getValue('rush_rules_path')
pass
def requestUMLSAPIKey(self, rows):
api_key = ConfigReader.getValue("api_key")
if api_key is None or len(api_key) == 0:
rows.append(
widgets.HTML(
value=
'<h4>Set up your Glove model</h4><p>In order to use word embedding, you need '
'to tell where the glove model locates:</p>'))
self.api_key_input = widgets.Text(value='',
placeholder='',
description='',
disabled=False)
rows.append(self.api_key_input)
rows += self.addSeparator()
def __init__(self, **kwargs):
self.sample_size = 0
self.previous_sampled_ids = kwargs['previous_sampled_ids']
self.dao = kwargs['dao']
self.dataset_id = 'origin_doc' if 'dataset_id' not in kwargs else kwargs[
'dataset_id']
self.ignore_case = True
self.whoosh_root = ConfigReader.getValue('whoosh/root_path')
self.grouped_ids = dict()
self.all_contain_ids = set()
self.available_not_contain = 0
self.new_available_not_contain = 0
self.new_ids = dict()
self.current_stats = dict()
pass
def requestUMLSAPIKey(self, rows):
api_key = ConfigReader.getValue("api_key")
if api_key is None or len(api_key) == 0:
rows.append(
widgets.HTML(
value='<h4>Set your API Key</h4><p>In order to use the UMLS synonym checking module, you need to set'
' up your API key: (<a href="https://www.nlm.nih.gov/research/umls/user_education/quick_tours/'
'UTS-API/UTS_REST_API_Authentication.html" target="_blank">How to get your API Key_at 01:12 from'
' beginning. </a>)</p><p>If you do not set the api key, the UMLS synonym extender will be '
'<b>skipped</b>.</p>'))
self.api_key_input = widgets.Text(value='',
placeholder='',
description='', disabled=False)
rows.append(self.api_key_input)
rows += self.addSeparator()
def __init__(self,
description='',
name=str(Step.global_id + 1),
sampler_cls: type = KeywordStratefiedSampler):
super().__init__(name=name)
self.toggle = widgets.ToggleButtons(
options=sample_options,
value=sample_options[-1],
description='What to do with previously sampled data? ',
style=dict(description_width='initial'),
button_style='info')
self.toggle.observe(self.onPreviousSampleHandleChange)
self.sample_size_input = widgets.BoundedIntText(
value=0,
min=0,
max=0,
step=1,
description='Total documents you want to sample:',
style=dict(description_width='initial'))
self.sample_size_input.observe(self.onSampleConfigChange)
self.sampler_cls = sampler_cls
self.sampled_summary = widgets.HTML(value='')
self.percent_slider = widgets.IntSlider(value=70,
min=0,
max=100,
step=5,
description='',
disabled=False,
continuous_update=False,
orientation='horizontal',
readout=True,
readout_format='d')
self.percent_slider.observe(self.onSampleConfigChange)
# save DOC_IDs that contain or not contain keywords filters (used in sampling strategy)
self.samples = {"contain": [], "notcontain": []}
self.box = None
self.data = {'docs': [], 'annos': OrderedDict()}
self.ready = False
# reset, continue, addmore,
self.move_next_option = ''
self.total = None
self.total_contains = None
self.un_reviewed = 0
self.sampler = None
self.samples = dict()
self.current_stats = dict()
self.max_threshold = ConfigReader.getValue("review/rb_model_threshold")
self.sample_sizes = dict()
def navigate(self, button):
if self.glove_path_input is not None:
self.saveGloveConfig()
if self.api_key_input is not None:
self.saveAPIKey()
else:
self.workflow.api_key = ConfigReader.getValue("api_key")
self.backgroundWork()
if button.description == 'ContinueReviewing':
self.workflow.to_continue = True
self.workflow.steps[1].start()
self.workflow.steps[1].complete()
else:
self.workflow.to_continue = False
self.workflow.steps[1].start()
pass
def init_real_time(self):
self.ml_classifier = self.ml_classifier_cls(
task_name=self.workflow.task_name)
self.learning_pace = ConfigReader.getValue("review/ml_learning_pace")
self.loop_workflow.filters = self.workflow.filters
self.readData()
if self.ml_classifier_cls.status == NotTrained:
self.backgroundTraining()
self.nlp = ReviewRBInit.nlp
self.matcher = ReviewRBInit.matcher
logMsg([doc.DOC_ID for doc in self.docs])
if self.docs is not None and len(
self.docs) > 0 and (self.loop_workflow is None
or len(self.loop_workflow.steps) == 0):
last_doc_pos = len(self.reviewed_docs) + 1 if len(
self.reviewed_docs) < len(self.docs) else len(
self.reviewed_docs)
for i in range(0, last_doc_pos):
doc = self.docs[i]
content = self.genContent(doc)
reviewed = False
if doc.DOC_ID in self.annos and self.annos[
doc.DOC_ID].REVIEWED_TYPE is not None:
prediction = self.annos[doc.DOC_ID].REVIEWED_TYPE
reviewed = True
else:
prediction = self.getPrediction(doc)
repeat_step = ReviewML(
description=content,
options=self.workflow.types,
value=prediction,
js=self.js,
master=self,
reviewed=reviewed,
button_style=('success' if reviewed else 'info'))
self.appendRepeatStep(repeat_step)
pass
from SmartAnno.utils.AnnotationTypeDef import AnnotationTypeDef
from SmartAnno.utils.KeywordsFiltering import KeywordsFiltering
from SmartAnno.gui.PreviousNextWidgets import PreviousNextHTML
from SmartAnno.utils.ReviewRBInit import ReviewRBInit
from SmartAnno.utils.ReviewRBLoop import ReviewRBLoop
from SmartAnno.utils.ReviewMLInit import ReviewMLInit
from SmartAnno.utils.ReviewMLLoop import ReviewMLLoop
from SmartAnno.models.logistic.LogisticBOWClassifiers import LogisticBOWClassifier
from SmartAnno.utils.DataSetChooser import DataSetChooser
logging.getLogger().setLevel(logging.DEBUG)
ConfigReader('../conf/smartanno_conf.json')
wf = Workflow(config_file=ConfigReader.config_file)
wf.api_key = ConfigReader.getValue("api_key")
wf.dao = Dao('sqlite+pysqlite:///../data/test.sqlite',
sqlalchemy_dao.POOL_DISABLED)
wf.task_name = 'language'
wf.append(
AnnotationTypeDef(
'<h3>Annotation types:</h3><p>List all the types you want to identify below. Each type per line.<br/>If you'
'have too many types, try set up them separately, so that you won't need to choose from a long list '
'for each sample. </p>',
name='types'))
wf.append(KeywordsFiltering(name='keywords'))
wf.append(
DataSetChooser(
name='dataset_chooser',
description='<h4>Choose which dateaset you want to use: </h4>'))
rb = ReviewRBInit(name="rb_review_init")
def restoreStatus(self):
status = ConfigReader.getValue('status/' + self.name)
if status is None or status == '':
status = 0
return status
from SmartAnno.utils.ConfigReader import ConfigReader
from SmartAnno.umls.UMLSFinder import UMLSFinder
ConfigReader()
umls = UMLSFinder(ConfigReader.getValue("api_key"),
sources=[],
filter_by_length=5,
max_query=50,
filter_by_contains=True)
print(umls.search("ketoacidosis"))
from SmartAnno.utils.ConfigReader import ConfigReader
from SmartAnno.db.ORMs import Filter
from SmartAnno.gui.Workflow import Workflow
from SmartAnno.utils.AnnotationTypeDef import AnnotationTypeDef
from SmartAnno.utils.IntroStep import IntroStep
from SmartAnno.utils.KeywordsFiltering import KeywordsFiltering
from SmartAnno.utils.KeywordsUMLSExtender import KeywordsUMLSExtender
from SmartAnno.utils.KeywordsUMLSExtenderSetup import KeywordsUMLSExtenderSetup
logging.getLogger().setLevel(logging.DEBUG)
ConfigReader('../conf/smartanno_conf.json')
wf = Workflow(config_file=ConfigReader.config_file)
wf.api_key = ConfigReader.getValue("api_key")
wf.dao = Dao('sqlite+pysqlite:///../data/test.sqlite', sqlalchemy_dao.POOL_DISABLED)
wf.task_name = 'language'
wf.append(AnnotationTypeDef(
'<h3>Annotation types:</h3><p>List all the types you want to identify below. Each type per line.<br/>If you'
'have too many types, try set up them separately, so that you won't need to choose from a long list '
'for each sample. </p>', name='types'))
wf.append(KeywordsFiltering(
name='keywords'))
wf.append(KeywordsUMLSExtenderSetup(name='umls_extender_setup'))
wf.append(KeywordsUMLSExtender(name='umls_extender', sources=ConfigReader.getValue("umls/sources"),
filter_by_length=ConfigReader.getValue("umls/filter_by_length"),
filter_by_contains=ConfigReader.getValue("umls/filter_by_contains"),
max_query=ConfigReader.getValue("umls/max_query")))
wf.append(
IntroStep('<h2>Welcome to SmartAnno!</h2><h4>First, let's import txt data from a directory. </h4>',
def pyRuSHSplitter(self, text):
rush = RuSH(ConfigReader.getValue('rush_rules_path'))
sentences = rush.segToSentenceSpans(text)
return [
text[sentence.begin:sentence.end].strip() for sentence in sentences
]