def __init__(self, nlpdef: NlpDefinition, cfgsection: str, commit: bool = False) -> None: """ Args: nlpdef: a :class:`crate_anon.nlp_manager.nlp_definition.NlpDefinition` cfgsection: the name of a CRATE NLP config file section (from which we may choose to get extra config information) commit: force a COMMIT whenever we insert data? You should specify this in multiprocess mode, or you may get database deadlocks. """ super().__init__(nlpdef=nlpdef, cfgsection=cfgsection, commit=commit, name="MedEx") if nlpdef is None: # only None for debugging! self._debug_mode = True self._tablename = self.classname().lower() self._max_external_prog_uses = 1 self._progenvsection = "" self._env = {} # type: Dict[str, str] progargs = "" else: self._debug_mode = False self._tablename = nlpdef.opt_str(self._sectionname, ProcessorConfigKeys.DESTTABLE, required=True) self._max_external_prog_uses = nlpdef.opt_int( self._sectionname, ProcessorConfigKeys.MAX_EXTERNAL_PROG_USES, default=0) self._progenvsection = nlpdef.opt_str( self._sectionname, ProcessorConfigKeys.PROGENVSECTION) if self._progenvsection: self._env = nlpdef.get_env_dict( full_sectionname(NlpConfigPrefixes.ENV, self._progenvsection), os.environ) else: self._env = os.environ.copy() self._env["NLPLOGTAG"] = nlpdef.get_logtag() or '.' # ... because passing a "-lt" switch with no parameter will make # CrateGatePipeline.java complain and stop progargs = nlpdef.opt_str(self._sectionname, ProcessorConfigKeys.PROGARGS, required=True) if USE_TEMP_DIRS: self._inputdir = tempfile.TemporaryDirectory() self._outputdir = tempfile.TemporaryDirectory() self._workingdir = tempfile.TemporaryDirectory() # ... these are autodeleted when the object goes out of scope; see # https://docs.python.org/3/library/tempfile.html # ... which manages it using weakref.finalize else: homedir = os.path.expanduser("~") self._inputdir = PseudoTempDir( os.path.join(homedir, "medextemp", "input")) mkdir_p(self._inputdir.name) self._outputdir = PseudoTempDir( os.path.join(homedir, "medextemp", "output")) mkdir_p(self._outputdir.name) self._workingdir = PseudoTempDir( os.path.join(homedir, "medextemp", "working")) mkdir_p(self._workingdir.name) formatted_progargs = progargs.format(**self._env) self._progargs = shlex.split(formatted_progargs) self._progargs.extend([ "-data_ready_signal", MEDEX_DATA_READY_SIGNAL, "-results_ready_signal", MEDEX_RESULTS_READY_SIGNAL, "-i", self._inputdir.name, "-o", self._outputdir.name, ]) self._n_uses = 0 self._pipe_encoding = 'utf8' self._file_encoding = 'utf8' self._p = None # the subprocess self._started = False
def __init__(self, nlpdef: NlpDefinition, cfgsection: str, commit: bool = False) -> None: super().__init__(nlpdef=nlpdef, cfgsection=cfgsection, commit=commit) self._tablename = nlpdef.opt_str(cfgsection, 'desttable', required=True) self._max_external_prog_uses = nlpdef.opt_int(cfgsection, 'max_external_prog_uses', default=0) self._progenvsection = nlpdef.opt_str(cfgsection, 'progenvsection') if self._progenvsection: self._env = nlpdef.get_env_dict(self._progenvsection, os.environ) else: self._env = os.environ.copy() self._env["NLPLOGTAG"] = nlpdef.get_logtag() or '.' # ... because passing a "-lt" switch with no parameter will make # CrateGatePipeline.java complain and stop if USE_TEMP_DIRS: self._inputdir = tempfile.TemporaryDirectory() self._outputdir = tempfile.TemporaryDirectory() self._workingdir = tempfile.TemporaryDirectory() # ... these are autodeleted when the object goes out of scope; see # https://docs.python.org/3/library/tempfile.html # ... which manages it using weakref.finalize else: homedir = os.path.expanduser("~") self._inputdir = AttrDict( name=os.path.join(homedir, "medextemp", "input")) mkdir_p(self._inputdir.name) self._outputdir = AttrDict( name=os.path.join(homedir, "medextemp", "output")) mkdir_p(self._outputdir.name) self._workingdir = AttrDict( name=os.path.join(homedir, "medextemp", "working")) mkdir_p(self._workingdir.name) progargs = nlpdef.opt_str(cfgsection, 'progargs', required=True) formatted_progargs = progargs.format(**self._env) self._progargs = shlex.split(formatted_progargs) self._progargs.extend([ "-data_ready_signal", MEDEX_DATA_READY_SIGNAL, "-results_ready_signal", MEDEX_RESULTS_READY_SIGNAL, "-i", self._inputdir.name, "-o", self._outputdir.name, ]) self._n_uses = 0 self._pipe_encoding = 'utf8' self._file_encoding = 'utf8' self._p = None # the subprocess self._started = False
def __init__(self, nlpdef: NlpDefinition, cfg_processor_name: str, commit: bool = False) -> None: """ Args: nlpdef: a :class:`crate_anon.nlp_manager.nlp_definition.NlpDefinition` cfg_processor_name: the name of a CRATE NLP config file section (from which we may choose to get extra config information) commit: force a COMMIT whenever we insert data? You should specify this in multiprocess mode, or you may get database deadlocks. """ super().__init__(nlpdef=nlpdef, cfg_processor_name=cfg_processor_name, commit=commit, friendly_name="GATE") if not nlpdef and not cfg_processor_name: # Debugging only self._debug_mode = True self._max_external_prog_uses = 0 self._input_terminator = 'input_terminator' self._output_terminator = 'output_terminator' typepairs = [] # type: List[str] self._progenvsection = '' progargs = '' logtag = '' else: self._debug_mode = False self._max_external_prog_uses = self._cfgsection.opt_int_positive( ProcessorConfigKeys.MAX_EXTERNAL_PROG_USES, default=0) self._input_terminator = self._cfgsection.opt_str( ProcessorConfigKeys.INPUT_TERMINATOR, required=True) self._output_terminator = self._cfgsection.opt_str( ProcessorConfigKeys.OUTPUT_TERMINATOR, required=True) typepairs = self._cfgsection.opt_strlist( ProcessorConfigKeys.OUTPUTTYPEMAP, required=True, lower=False) self._progenvsection = self._cfgsection.opt_str( ProcessorConfigKeys.PROGENVSECTION) progargs = self._cfgsection.opt_str(ProcessorConfigKeys.PROGARGS, required=True) logtag = nlpdef.logtag or '.' self._outputtypemap = {} # type: Dict[str, OutputUserConfig] self._type_to_tablename = {} # type: Dict[str, str] for annottype, outputsection in chunks(typepairs, 2): annottype = annottype.lower() c = OutputUserConfig(nlpdef.parser, outputsection) self._outputtypemap[annottype] = c self._type_to_tablename[annottype] = c.dest_tablename if self._progenvsection: # noinspection PyTypeChecker self._env = nlpdef.get_env_dict(self._progenvsection, os.environ) else: self._env = os.environ.copy() self._env["NLPLOGTAG"] = logtag # ... We have ensured that this is not empty for real use, because # passing a "-lt" switch with no parameter will make # CrateGatePipeline.java complain and stop. The environment variable # is read via the "progargs" config argument, as follows. formatted_progargs = progargs.format(**self._env) self._progargs = shlex.split(formatted_progargs) self._n_uses = 0 self._pipe_encoding = 'utf8' self._p = None # the subprocess self._started = False # Sanity checks for ty, tn in self._type_to_tablename.items(): assert len(tn) <= MAX_SQL_FIELD_LEN, ( f"Table name too long (max {MAX_SQL_FIELD_LEN} characters)")
def __init__(self, nlpdef: NlpDefinition, cfgsection: str, commit: bool = False) -> None: super().__init__(nlpdef=nlpdef, cfgsection=cfgsection, commit=commit) if not nlpdef and not cfgsection: # Debugging only self._max_external_prog_uses = 0 self._input_terminator = 'input_terminator' self._output_terminator = 'output_terminator' typepairs = [] # type: List[str] self._progenvsection = '' progargs = '' logtag = '' else: self._max_external_prog_uses = nlpdef.opt_int( cfgsection, 'max_external_prog_uses', default=0) self._input_terminator = nlpdef.opt_str(cfgsection, 'input_terminator', required=True) self._output_terminator = nlpdef.opt_str(cfgsection, 'output_terminator', required=True) typepairs = nlpdef.opt_strlist(cfgsection, 'outputtypemap', required=True, lower=False) self._progenvsection = nlpdef.opt_str(cfgsection, 'progenvsection') progargs = nlpdef.opt_str(cfgsection, 'progargs', required=True) logtag = nlpdef.get_logtag() or '.' self._outputtypemap = {} # type: Dict[str, OutputUserConfig] self._type_to_tablename = {} # type: Dict[str, str] for c in chunks(typepairs, 2): annottype = c[0] outputsection = c[1] if annottype != annottype.lower(): raise Exception( "Section {}: annotation types in outputtypemap must be in " "lower case: change {}".format(cfgsection, annottype)) # log.critical(outputsection) c = OutputUserConfig(nlpdef.get_parser(), outputsection) self._outputtypemap[annottype] = c self._type_to_tablename[annottype] = c.get_tablename() if self._progenvsection: self._env = nlpdef.get_env_dict(self._progenvsection, os.environ) else: self._env = os.environ.copy() self._env["NLPLOGTAG"] = logtag # ... because passing a "-lt" switch with no parameter will make # CrateGatePipeline.java complain and stop formatted_progargs = progargs.format(**self._env) self._progargs = shlex.split(formatted_progargs) self._n_uses = 0 self._pipe_encoding = 'utf8' self._p = None # the subprocess self._started = False # Sanity checks for ty, tn in self._type_to_tablename.items(): assert len(tn) <= MAX_SQL_FIELD_LEN, ( "Table name too long (max {} characters)".format( MAX_SQL_FIELD_LEN))
def __init__(self, nlpdef: NlpDefinition, cfgsection: str, commit: bool = False) -> None: """ Args: nlpdef: a :class:`crate_anon.nlp_manager.nlp_definition.NlpDefinition` cfgsection: the name of a CRATE NLP config file section (from which we may choose to get extra config information) commit: force a COMMIT whenever we insert data? You should specify this in multiprocess mode, or you may get database deadlocks. """ super().__init__(nlpdef=nlpdef, cfgsection=cfgsection, commit=commit, name="GATE") if not nlpdef and not cfgsection: # Debugging only self._debug_mode = True self._max_external_prog_uses = 0 self._input_terminator = 'input_terminator' self._output_terminator = 'output_terminator' typepairs = [] # type: List[str] self._progenvsection = '' progargs = '' logtag = '' else: self._debug_mode = False self._max_external_prog_uses = nlpdef.opt_int( self._sectionname, ProcessorConfigKeys.MAX_EXTERNAL_PROG_USES, default=0) self._input_terminator = nlpdef.opt_str( self._sectionname, ProcessorConfigKeys.INPUT_TERMINATOR, required=True) self._output_terminator = nlpdef.opt_str( self._sectionname, ProcessorConfigKeys.OUTPUT_TERMINATOR, required=True) typepairs = nlpdef.opt_strlist( self._sectionname, ProcessorConfigKeys.OUTPUTTYPEMAP, required=True, lower=False) self._progenvsection = nlpdef.opt_str( self._sectionname, ProcessorConfigKeys.PROGENVSECTION) progargs = nlpdef.opt_str( self._sectionname, ProcessorConfigKeys.PROGARGS, required=True) logtag = nlpdef.get_logtag() or '.' self._outputtypemap = {} # type: Dict[str, OutputUserConfig] self._type_to_tablename = {} # type: Dict[str, str] for c in chunks(typepairs, 2): annottype = c[0] outputsection = c[1] # 2018-03-27: not clear why we need to force the user to specify # in lower case! We just said it's case-insensitive. So ditch this: # # if annottype != annottype.lower(): # raise Exception( # "Section {}: annotation types in outputtypemap must be in " # noqa # "lower case: change {}".format(cfgsection, annottype)) # # and add this: annottype = annottype.lower() # log.critical(outputsection) c = OutputUserConfig(nlpdef.get_parser(), outputsection) self._outputtypemap[annottype] = c self._type_to_tablename[annottype] = c.get_tablename() if self._progenvsection: self._env = nlpdef.get_env_dict( full_sectionname(NlpConfigPrefixes.ENV, self._progenvsection), os.environ) else: self._env = os.environ.copy() self._env["NLPLOGTAG"] = logtag # ... We have ensured that this is not empty for real use, because # passing a "-lt" switch with no parameter will make # CrateGatePipeline.java complain and stop. The environment variable # is read via the "progargs" config argument, as follows. formatted_progargs = progargs.format(**self._env) self._progargs = shlex.split(formatted_progargs) self._n_uses = 0 self._pipe_encoding = 'utf8' self._p = None # the subprocess self._started = False # Sanity checks for ty, tn in self._type_to_tablename.items(): assert len(tn) <= MAX_SQL_FIELD_LEN, ( f"Table name too long (max {MAX_SQL_FIELD_LEN} characters)")