def __init__(self,
nlpdef: NlpDefinition,
cfgsection: str,
variable: str,
target_unit: str,
regex_str_for_debugging: str,
commit: bool = False) -> None:
super().__init__(nlpdef=nlpdef, cfgsection=cfgsection, commit=commit)
self.variable = variable
self.target_unit = target_unit
self.regex_str_for_debugging = regex_str_for_debugging
if nlpdef is None: # only None for debugging!
self.tablename = ''
self.assume_preferred_unit = True
else:
self.tablename = nlpdef.opt_str(
cfgsection, 'desttable', required=True)
self.assume_preferred_unit = nlpdef.opt_bool(
cfgsection, 'assume_preferred_unit', default=True)
# Sanity checks
assert len(self.variable) <= MAX_SQL_FIELD_LEN, (
"Variable name too long (max {} characters)".format(
MAX_SQL_FIELD_LEN))
def drop_remake(progargs,
nlpdef: NlpDefinition,
incremental: bool = False,
skipdelete: bool = False) -> None:
"""
Drop output tables and recreate them.
"""
# Not parallel.
# -------------------------------------------------------------------------
# 1. Progress database
# -------------------------------------------------------------------------
progengine = nlpdef.get_progdb_engine()
if not incremental:
log.debug("Dropping progress tables")
NlpRecord.__table__.drop(progengine, checkfirst=True)
log.info("Creating progress table (with index)")
NlpRecord.__table__.create(progengine, checkfirst=True)
# -------------------------------------------------------------------------
# 2. Output database(s)
# -------------------------------------------------------------------------
pretty_names = [] # type: List[str]
for processor in nlpdef.get_processors():
new_pretty_names = processor.make_tables(drop_first=not incremental)
for npn in new_pretty_names:
if npn in pretty_names:
log.warning("An NLP processor has tried to re-make a table "
"made by one of its colleagues: {}".format(npn))
pretty_names.extend(new_pretty_names)
# -------------------------------------------------------------------------
# 3. Delete WHERE NOT IN for incremental
# -------------------------------------------------------------------------
for ifconfig in nlpdef.get_ifconfigs():
with MultiTimerContext(timer, TIMING_DELETE_WHERE_NO_SOURCE):
if incremental:
if not skipdelete:
delete_where_no_source(
nlpdef,
ifconfig,
report_every=progargs.report_every_fast,
chunksize=progargs.chunksize)
else: # full
ifconfig.delete_all_progress_records()
# -------------------------------------------------------------------------
# 4. Overall commit (superfluous)
# -------------------------------------------------------------------------
nlpdef.commit_all()
def show_dest_counts(nlpdef: NlpDefinition) -> None:
"""
Show the number of records in all destination tables.
"""
print("DESTINATION TABLE RECORD COUNTS:")
counts = [] # type: List[Tuple[str, int]]
for processor in nlpdef.get_processors():
session = processor.get_session()
dbname = processor.get_dbname()
for tablename in processor.get_tablenames():
n = count_star(session, tablename)
counts.append(("DESTINATION: {}.{}".format(dbname, tablename), n))
print_record_counts(counts)
def show_source_counts(nlpdef: NlpDefinition) -> None:
"""
Show the number of records in all source tables.
"""
print("SOURCE TABLE RECORD COUNTS:")
counts = [] # type: List[Tuple[str, int]]
for ifconfig in nlpdef.get_ifconfigs():
session = ifconfig.get_source_session()
dbname = ifconfig.get_srcdb()
tablename = ifconfig.get_srctable()
n = count_star(session, tablename)
counts.append(("{}.{}".format(dbname, tablename), n))
print_record_counts(counts)
def __init__(self, nlpdef: NlpDefinition) -> None:
"""
Args:
nlpdef:
:class:`crate_anon.nlp_manager.nlp_definition.NlpDefinition`
"""
self._nlpdef = nlpdef
self._cloudcfg = nlpdef.get_cloud_config_or_raise()
self._nlpdef_sectionname = full_sectionname(NlpConfigPrefixes.NLPDEF,
self._nlpdef.get_name())
self._auth = (self._cloudcfg.username, self._cloudcfg.password)
self._post = self._internal_post
self.cookies = None # type: Optional[CookieJar]
def __init__(self,
nlpdef: NlpDefinition) \
-> None:
"""
Args:
nlpdef:
a :class:`crate_anon.nlp_manager.nlp_definition.NlpDefinition`
"""
self.nlpdef = nlpdef
# Convenience member for our users:
self.cloudcfg = nlpdef.get_cloud_config_or_raise()
self._remote_processors = None # type: Optional[List[ServerProcessor]]
self._local_processors = None # type: Optional[List[Cloud]]
self._configure_local_processors()
def __init__(self,
nlpdef: NlpDefinition,
cfgsection: str,
regex_str_list: List[str],
validated_variable: str,
commit: bool = False) -> None:
"""
This class operates with compiled regexes having this group format:
- variable
"""
super().__init__(nlpdef=nlpdef, cfgsection=cfgsection, commit=commit)
self.regex_str_list = regex_str_list # for debugging only
self.compiled_regex_list = [compile_regex(r) for r in regex_str_list]
self.variable = "{}_validator".format(validated_variable)
self.NAME = self.variable
if nlpdef is None: # only None for debugging!
self.tablename = ''
else:
self.tablename = nlpdef.opt_str(
cfgsection, 'desttable', required=True)
def __init__(self,
nlpdef: NlpDefinition,
cfgsection: str,
commit: bool = False) -> None:
"""
Args:
nlpdef:
a :class:`crate_anon.nlp_manager.nlp_definition.NlpDefinition`
cfgsection:
the name of a CRATE NLP config file section (from which we may
choose to get extra config information)
commit:
force a COMMIT whenever we insert data? You should specify this
in multiprocess mode, or you may get database deadlocks.
"""
super().__init__(nlpdef=nlpdef,
cfgsection=cfgsection,
commit=commit,
name="MedEx")
if nlpdef is None: # only None for debugging!
self._debug_mode = True
self._tablename = self.classname().lower()
self._max_external_prog_uses = 1
self._progenvsection = ""
self._env = {} # type: Dict[str, str]
progargs = ""
else:
self._debug_mode = False
self._tablename = nlpdef.opt_str(self._sectionname,
ProcessorConfigKeys.DESTTABLE,
required=True)
self._max_external_prog_uses = nlpdef.opt_int(
self._sectionname,
ProcessorConfigKeys.MAX_EXTERNAL_PROG_USES,
default=0)
self._progenvsection = nlpdef.opt_str(
self._sectionname, ProcessorConfigKeys.PROGENVSECTION)
if self._progenvsection:
self._env = nlpdef.get_env_dict(
full_sectionname(NlpConfigPrefixes.ENV,
self._progenvsection), os.environ)
else:
self._env = os.environ.copy()
self._env["NLPLOGTAG"] = nlpdef.get_logtag() or '.'
# ... because passing a "-lt" switch with no parameter will make
# CrateGatePipeline.java complain and stop
progargs = nlpdef.opt_str(self._sectionname,
ProcessorConfigKeys.PROGARGS,
required=True)
if USE_TEMP_DIRS:
self._inputdir = tempfile.TemporaryDirectory()
self._outputdir = tempfile.TemporaryDirectory()
self._workingdir = tempfile.TemporaryDirectory()
# ... these are autodeleted when the object goes out of scope; see
# https://docs.python.org/3/library/tempfile.html
# ... which manages it using weakref.finalize
else:
homedir = os.path.expanduser("~")
self._inputdir = PseudoTempDir(
os.path.join(homedir, "medextemp", "input"))
mkdir_p(self._inputdir.name)
self._outputdir = PseudoTempDir(
os.path.join(homedir, "medextemp", "output"))
mkdir_p(self._outputdir.name)
self._workingdir = PseudoTempDir(
os.path.join(homedir, "medextemp", "working"))
mkdir_p(self._workingdir.name)
formatted_progargs = progargs.format(**self._env)
self._progargs = shlex.split(formatted_progargs)
self._progargs.extend([
"-data_ready_signal",
MEDEX_DATA_READY_SIGNAL,
"-results_ready_signal",
MEDEX_RESULTS_READY_SIGNAL,
"-i",
self._inputdir.name,
"-o",
self._outputdir.name,
])
self._n_uses = 0
self._pipe_encoding = 'utf8'
self._file_encoding = 'utf8'
self._p = None # the subprocess
self._started = False
def __init__(self,
nlpdef: NlpDefinition,
cfgsection: str,
commit: bool = False) -> None:
super().__init__(nlpdef=nlpdef, cfgsection=cfgsection, commit=commit)
self._tablename = nlpdef.opt_str(cfgsection,
'desttable',
required=True)
self._max_external_prog_uses = nlpdef.opt_int(cfgsection,
'max_external_prog_uses',
default=0)
self._progenvsection = nlpdef.opt_str(cfgsection, 'progenvsection')
if self._progenvsection:
self._env = nlpdef.get_env_dict(self._progenvsection, os.environ)
else:
self._env = os.environ.copy()
self._env["NLPLOGTAG"] = nlpdef.get_logtag() or '.'
# ... because passing a "-lt" switch with no parameter will make
# CrateGatePipeline.java complain and stop
if USE_TEMP_DIRS:
self._inputdir = tempfile.TemporaryDirectory()
self._outputdir = tempfile.TemporaryDirectory()
self._workingdir = tempfile.TemporaryDirectory()
# ... these are autodeleted when the object goes out of scope; see
# https://docs.python.org/3/library/tempfile.html
# ... which manages it using weakref.finalize
else:
homedir = os.path.expanduser("~")
self._inputdir = AttrDict(
name=os.path.join(homedir, "medextemp", "input"))
mkdir_p(self._inputdir.name)
self._outputdir = AttrDict(
name=os.path.join(homedir, "medextemp", "output"))
mkdir_p(self._outputdir.name)
self._workingdir = AttrDict(
name=os.path.join(homedir, "medextemp", "working"))
mkdir_p(self._workingdir.name)
progargs = nlpdef.opt_str(cfgsection, 'progargs', required=True)
formatted_progargs = progargs.format(**self._env)
self._progargs = shlex.split(formatted_progargs)
self._progargs.extend([
"-data_ready_signal",
MEDEX_DATA_READY_SIGNAL,
"-results_ready_signal",
MEDEX_RESULTS_READY_SIGNAL,
"-i",
self._inputdir.name,
"-o",
self._outputdir.name,
])
self._n_uses = 0
self._pipe_encoding = 'utf8'
self._file_encoding = 'utf8'
self._p = None # the subprocess
self._started = False
def __init__(self,
nlpdef: NlpDefinition,
cfgsection: str,
commit: bool = False) -> None:
super().__init__(nlpdef=nlpdef, cfgsection=cfgsection, commit=commit)
if not nlpdef and not cfgsection:
# Debugging only
self._max_external_prog_uses = 0
self._input_terminator = 'input_terminator'
self._output_terminator = 'output_terminator'
typepairs = [] # type: List[str]
self._progenvsection = ''
progargs = ''
logtag = ''
else:
self._max_external_prog_uses = nlpdef.opt_int(
cfgsection, 'max_external_prog_uses', default=0)
self._input_terminator = nlpdef.opt_str(cfgsection,
'input_terminator',
required=True)
self._output_terminator = nlpdef.opt_str(cfgsection,
'output_terminator',
required=True)
typepairs = nlpdef.opt_strlist(cfgsection,
'outputtypemap',
required=True,
lower=False)
self._progenvsection = nlpdef.opt_str(cfgsection, 'progenvsection')
progargs = nlpdef.opt_str(cfgsection, 'progargs', required=True)
logtag = nlpdef.get_logtag() or '.'
self._outputtypemap = {} # type: Dict[str, OutputUserConfig]
self._type_to_tablename = {} # type: Dict[str, str]
for c in chunks(typepairs, 2):
annottype = c[0]
outputsection = c[1]
if annottype != annottype.lower():
raise Exception(
"Section {}: annotation types in outputtypemap must be in "
"lower case: change {}".format(cfgsection, annottype))
# log.critical(outputsection)
c = OutputUserConfig(nlpdef.get_parser(), outputsection)
self._outputtypemap[annottype] = c
self._type_to_tablename[annottype] = c.get_tablename()
if self._progenvsection:
self._env = nlpdef.get_env_dict(self._progenvsection, os.environ)
else:
self._env = os.environ.copy()
self._env["NLPLOGTAG"] = logtag
# ... because passing a "-lt" switch with no parameter will make
# CrateGatePipeline.java complain and stop
formatted_progargs = progargs.format(**self._env)
self._progargs = shlex.split(formatted_progargs)
self._n_uses = 0
self._pipe_encoding = 'utf8'
self._p = None # the subprocess
self._started = False
# Sanity checks
for ty, tn in self._type_to_tablename.items():
assert len(tn) <= MAX_SQL_FIELD_LEN, (
"Table name too long (max {} characters)".format(
MAX_SQL_FIELD_LEN))
def __init__(self,
nlpdef: NlpDefinition,
cfg_processor_name: str,
commit: bool = False) -> None:
"""
Args:
nlpdef:
a :class:`crate_anon.nlp_manager.nlp_definition.NlpDefinition`
cfg_processor_name:
the name of a CRATE NLP config file section (from which we may
choose to get extra config information)
commit:
force a COMMIT whenever we insert data? You should specify this
in multiprocess mode, or you may get database deadlocks.
"""
super().__init__(nlpdef=nlpdef,
cfg_processor_name=cfg_processor_name,
commit=commit,
friendly_name="GATE")
if not nlpdef and not cfg_processor_name:
# Debugging only
self._debug_mode = True
self._max_external_prog_uses = 0
self._input_terminator = 'input_terminator'
self._output_terminator = 'output_terminator'
typepairs = [] # type: List[str]
self._progenvsection = ''
progargs = ''
logtag = ''
else:
self._debug_mode = False
self._max_external_prog_uses = self._cfgsection.opt_int_positive(
ProcessorConfigKeys.MAX_EXTERNAL_PROG_USES, default=0)
self._input_terminator = self._cfgsection.opt_str(
ProcessorConfigKeys.INPUT_TERMINATOR, required=True)
self._output_terminator = self._cfgsection.opt_str(
ProcessorConfigKeys.OUTPUT_TERMINATOR, required=True)
typepairs = self._cfgsection.opt_strlist(
ProcessorConfigKeys.OUTPUTTYPEMAP, required=True, lower=False)
self._progenvsection = self._cfgsection.opt_str(
ProcessorConfigKeys.PROGENVSECTION)
progargs = self._cfgsection.opt_str(ProcessorConfigKeys.PROGARGS,
required=True)
logtag = nlpdef.logtag or '.'
self._outputtypemap = {} # type: Dict[str, OutputUserConfig]
self._type_to_tablename = {} # type: Dict[str, str]
for annottype, outputsection in chunks(typepairs, 2):
annottype = annottype.lower()
c = OutputUserConfig(nlpdef.parser, outputsection)
self._outputtypemap[annottype] = c
self._type_to_tablename[annottype] = c.dest_tablename
if self._progenvsection:
# noinspection PyTypeChecker
self._env = nlpdef.get_env_dict(self._progenvsection, os.environ)
else:
self._env = os.environ.copy()
self._env["NLPLOGTAG"] = logtag
# ... We have ensured that this is not empty for real use, because
# passing a "-lt" switch with no parameter will make
# CrateGatePipeline.java complain and stop. The environment variable
# is read via the "progargs" config argument, as follows.
formatted_progargs = progargs.format(**self._env)
self._progargs = shlex.split(formatted_progargs)
self._n_uses = 0
self._pipe_encoding = 'utf8'
self._p = None # the subprocess
self._started = False
# Sanity checks
for ty, tn in self._type_to_tablename.items():
assert len(tn) <= MAX_SQL_FIELD_LEN, (
f"Table name too long (max {MAX_SQL_FIELD_LEN} characters)")
def __init__(self,
nlpdef: NlpDefinition,
cfgsection: str,
commit: bool = False) -> None:
"""
Args:
nlpdef:
a :class:`crate_anon.nlp_manager.nlp_definition.NlpDefinition`
cfgsection:
the name of a CRATE NLP config file section (from which we may
choose to get extra config information)
commit:
force a COMMIT whenever we insert data? You should specify this
in multiprocess mode, or you may get database deadlocks.
"""
super().__init__(nlpdef=nlpdef, cfgsection=cfgsection, commit=commit,
name="GATE")
if not nlpdef and not cfgsection:
# Debugging only
self._debug_mode = True
self._max_external_prog_uses = 0
self._input_terminator = 'input_terminator'
self._output_terminator = 'output_terminator'
typepairs = [] # type: List[str]
self._progenvsection = ''
progargs = ''
logtag = ''
else:
self._debug_mode = False
self._max_external_prog_uses = nlpdef.opt_int(
self._sectionname, ProcessorConfigKeys.MAX_EXTERNAL_PROG_USES,
default=0)
self._input_terminator = nlpdef.opt_str(
self._sectionname, ProcessorConfigKeys.INPUT_TERMINATOR,
required=True)
self._output_terminator = nlpdef.opt_str(
self._sectionname, ProcessorConfigKeys.OUTPUT_TERMINATOR,
required=True)
typepairs = nlpdef.opt_strlist(
self._sectionname, ProcessorConfigKeys.OUTPUTTYPEMAP,
required=True, lower=False)
self._progenvsection = nlpdef.opt_str(
self._sectionname, ProcessorConfigKeys.PROGENVSECTION)
progargs = nlpdef.opt_str(
self._sectionname, ProcessorConfigKeys.PROGARGS,
required=True)
logtag = nlpdef.get_logtag() or '.'
self._outputtypemap = {} # type: Dict[str, OutputUserConfig]
self._type_to_tablename = {} # type: Dict[str, str]
for c in chunks(typepairs, 2):
annottype = c[0]
outputsection = c[1]
# 2018-03-27: not clear why we need to force the user to specify
# in lower case! We just said it's case-insensitive. So ditch this:
#
# if annottype != annottype.lower():
# raise Exception(
# "Section {}: annotation types in outputtypemap must be in " # noqa
# "lower case: change {}".format(cfgsection, annottype))
#
# and add this:
annottype = annottype.lower()
# log.critical(outputsection)
c = OutputUserConfig(nlpdef.get_parser(), outputsection)
self._outputtypemap[annottype] = c
self._type_to_tablename[annottype] = c.get_tablename()
if self._progenvsection:
self._env = nlpdef.get_env_dict(
full_sectionname(NlpConfigPrefixes.ENV,
self._progenvsection),
os.environ)
else:
self._env = os.environ.copy()
self._env["NLPLOGTAG"] = logtag
# ... We have ensured that this is not empty for real use, because
# passing a "-lt" switch with no parameter will make
# CrateGatePipeline.java complain and stop. The environment variable
# is read via the "progargs" config argument, as follows.
formatted_progargs = progargs.format(**self._env)
self._progargs = shlex.split(formatted_progargs)
self._n_uses = 0
self._pipe_encoding = 'utf8'
self._p = None # the subprocess
self._started = False
# Sanity checks
for ty, tn in self._type_to_tablename.items():
assert len(tn) <= MAX_SQL_FIELD_LEN, (
f"Table name too long (max {MAX_SQL_FIELD_LEN} characters)")
def main() -> None:
"""
Command-line entry point.
"""
version = "Version {} ({})".format(VERSION, VERSION_DATE)
description = "NLP manager. {version}. By Rudolf Cardinal.".format(
version=version)
parser = argparse.ArgumentParser(
description=description,
formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument("--version", action="version", version=version)
parser.add_argument("--config",
help="Config file (overriding environment "
"variable {})".format(NLP_CONFIG_ENV_VAR))
parser.add_argument('--verbose',
'-v',
action='store_true',
help="Be verbose (use twice for extra verbosity)")
parser.add_argument("--nlpdef",
nargs="?",
default=None,
help="NLP definition name (from config file)")
parser.add_argument('--report_every_fast',
nargs="?",
type=int,
default=DEFAULT_REPORT_EVERY,
help="Report insert progress (for fast operations) "
"every n rows in verbose "
"mode (default {})".format(DEFAULT_REPORT_EVERY))
parser.add_argument('--report_every_nlp',
nargs="?",
type=int,
default=DEFAULT_REPORT_EVERY_NLP,
help="Report progress for NLP every n rows in verbose "
"mode (default "
"{})".format(DEFAULT_REPORT_EVERY_NLP))
parser.add_argument('--chunksize',
nargs="?",
type=int,
default=DEFAULT_CHUNKSIZE,
help="Number of records copied in a chunk when copying"
" PKs from one database to another"
" (default {})".format(DEFAULT_CHUNKSIZE))
parser.add_argument("--process",
nargs="?",
type=int,
default=0,
help="For multiprocess mode: specify process number")
parser.add_argument("--nprocesses",
nargs="?",
type=int,
default=1,
help="For multiprocess mode: specify "
"total number of processes (launched somehow, of "
"which this is to be one)")
parser.add_argument("--processcluster",
default="",
help="Process cluster name")
parser.add_argument("--democonfig",
action="store_true",
help="Print a demo config file")
parser.add_argument("--listprocessors",
action="store_true",
help="Show possible built-in NLP processor names")
parser.add_argument("--describeprocessors",
action="store_true",
help="Show details of built-in NLP processors")
parser.add_argument("--showinfo",
required=False,
nargs='?',
metavar="NLP_CLASS_NAME",
help="Show detailed information for a parser")
parser.add_argument("--count",
action="store_true",
help="Count records in source/destination databases, "
"then stop")
mode_group = parser.add_mutually_exclusive_group()
mode_group.add_argument("-i",
"--incremental",
dest="incremental",
action="store_true",
help="Process only new/changed information, where "
"possible (* default)")
mode_group.add_argument("-f",
"--full",
dest="incremental",
action="store_false",
help="Drop and remake everything")
parser.set_defaults(incremental=True)
parser.add_argument("--dropremake",
action="store_true",
help="Drop/remake destination tables only")
parser.add_argument("--skipdelete",
dest="skipdelete",
action="store_true",
help="For incremental updates, skip deletion of rows "
"present in the destination but not the source")
parser.add_argument("--nlp",
action="store_true",
help="Perform NLP processing only")
parser.add_argument("--echo", action="store_true", help="Echo SQL")
parser.add_argument("--timing",
action="store_true",
help="Show detailed timing breakdown")
args = parser.parse_args()
# Validate args
if args.nprocesses < 1:
raise ValueError("--nprocesses must be >=1")
if args.process < 0 or args.process >= args.nprocesses:
raise ValueError(
"--process argument must be from 0 to (nprocesses - 1) inclusive")
if args.config:
os.environ[NLP_CONFIG_ENV_VAR] = args.config
# Verbosity and logging
mynames = [] # type: List[str]
if args.processcluster:
mynames.append(args.processcluster)
if args.nprocesses > 1:
mynames.append("proc{}".format(args.process))
loglevel = logging.DEBUG if args.verbose else logging.INFO
rootlogger = logging.getLogger()
configure_logger_for_colour(rootlogger, level=loglevel, extranames=mynames)
# -------------------------------------------------------------------------
# Demo config?
if args.democonfig:
print(DEMO_CONFIG)
return
# List or describe processors?
if args.listprocessors:
print("\n".join(possible_processor_names()))
return
if args.describeprocessors:
print(possible_processor_table())
return
if args.showinfo:
parser = get_nlp_parser_debug_instance(args.showinfo)
if parser:
print("Info for class {}:\n".format(args.showinfo))
parser.print_info()
else:
print("No such processor class: {}".format(args.showinfo))
return
# Otherwise, we need a valid NLP definition.
if args.nlpdef is None:
raise ValueError("Must specify nlpdef parameter (unless --democonfig, "
"--listprocessors, or --describeprocessors used)")
everything = not any([args.dropremake, args.nlp])
# Report args
log.debug("arguments: {}".format(args))
# Load/validate config
config = NlpDefinition(args.nlpdef,
logtag="_".join(mynames).replace(" ", "_"))
config.set_echo(args.echo)
# Count only?
if args.count:
show_source_counts(config)
show_dest_counts(config)
return
# -------------------------------------------------------------------------
log.info("Starting: incremental={}".format(args.incremental))
start = get_now_utc()
timer.set_timing(args.timing, reset=True)
# 1. Drop/remake tables. Single-tasking only.
with MultiTimerContext(timer, TIMING_DROP_REMAKE):
if args.dropremake or everything:
drop_remake(args,
config,
incremental=args.incremental,
skipdelete=args.skipdelete)
# From here, in a multiprocessing environment, trap any errors simply so
# we can report the process number clearly.
# 2. NLP
if args.nlp or everything:
try:
process_nlp(config,
incremental=args.incremental,
report_every=args.report_every_nlp,
tasknum=args.process,
ntasks=args.nprocesses)
except Exception as exc:
log.critical("TERMINAL ERROR FROM THIS PROCESS") # so we see proc#
die(exc)
log.info("Finished")
end = get_now_utc()
time_taken = end - start
log.info("Time taken: {:.3f} seconds".format(time_taken.total_seconds()))
if args.timing:
timer.report()
def __init__(self,
nlpdef: NlpDefinition,
cfgsection: str,
variable_name: str, # e.g. "MMSE"
variable_regex_str: str, # e.g. regex for MMSE
expected_denominator: int,
numerator_text_fieldname: str = "numerator_text",
numerator_fieldname: str = "numerator",
denominator_text_fieldname: str = "denominator_text",
denominator_fieldname: str = "denominator",
correct_numerator_fieldname: str = None, # default below
take_absolute: bool = True,
commit: bool = False,
debug: bool = False) -> None:
"""
This class operates with compiled regexes having this group format:
- quantity_regex_str: e.g. to find "MMSE"
"""
self.variable_name = variable_name
assert(expected_denominator > 0)
self.expected_denominator = expected_denominator
self.numerator_text_fieldname = numerator_text_fieldname
self.numerator_fieldname = numerator_fieldname
self.denominator_text_fieldname = denominator_text_fieldname
self.denominator_fieldname = denominator_fieldname
self.correct_numerator_fieldname = (
correct_numerator_fieldname or
"out_of_{}".format(expected_denominator))
self.take_absolute = take_absolute
super().__init__(nlpdef=nlpdef,
cfgsection=cfgsection,
commit=commit)
if nlpdef is None: # only None for debugging!
self.tablename = ''
else:
self.tablename = nlpdef.opt_str(
cfgsection, 'desttable', required=True)
regex_str = r"""
( {variable} ) # 1. group for variable (thing being measured)
{OPTIONAL_RESULTS_IGNORABLES}
{SCORE}? # optional "score" or similar
{OPTIONAL_RESULTS_IGNORABLES}
( {TENSE_INDICATOR} )? # 2. optional group for tense indicator
{OPTIONAL_RESULTS_IGNORABLES}
( {RELATION} )? # 3. optional group for relation
{OPTIONAL_RESULTS_IGNORABLES}
( {SIGNED_FLOAT} ) # 4. group for numerator
(?: # optional "/ denominator"
\s* {OUT_OF_SEPARATOR} \s*
( {UNSIGNED_INTEGER} ) # 5. group for denominator
)?
""".format( # noqa
variable=variable_regex_str,
OPTIONAL_RESULTS_IGNORABLES=OPTIONAL_RESULTS_IGNORABLES,
SCORE=SCORE,
TENSE_INDICATOR=TENSE_INDICATOR,
RELATION=RELATION,
SIGNED_FLOAT=SIGNED_FLOAT,
OUT_OF_SEPARATOR=OUT_OF_SEPARATOR,
UNSIGNED_INTEGER=UNSIGNED_INTEGER,
)
if debug:
print("Regex for {}: {}".format(type(self).__name__, regex_str))
self.regex_str = regex_str
self.compiled_regex = compile_regex(regex_str)
def process_nlp(nlpdef: NlpDefinition,
incremental: bool = False,
report_every: int = DEFAULT_REPORT_EVERY_NLP,
tasknum: int = 0,
ntasks: int = 1) -> None:
"""
Main NLP processing function. Fetch text, send it to the GATE app
(storing the results), and make a note in the progress database.
"""
log.info(SEP + "NLP")
session = nlpdef.get_progdb_session()
for ifconfig in nlpdef.get_ifconfigs():
i = 0 # record count within this process
recnum = tasknum # record count overall
totalcount = ifconfig.get_count() # total number of records in table
for text, other_values in ifconfig.gen_text(tasknum=tasknum,
ntasks=ntasks):
i += 1
pkval = other_values[FN_SRCPKVAL]
pkstr = other_values[FN_SRCPKSTR]
if report_every and i % report_every == 0:
log.info(
"Processing {db}.{t}.{c}, PK: {pkf}={pkv} "
"({overall}record {approx}{recnum}/{totalcount})"
"{thisproc}".format(
db=other_values[FN_SRCDB],
t=other_values[FN_SRCTABLE],
c=other_values[FN_SRCFIELD],
pkf=other_values[FN_SRCPKFIELD],
pkv=pkstr if pkstr else pkval,
overall="overall " if ntasks > 1 else "",
approx="~" if pkstr and ntasks > 1 else "",
# ... string hashing means approx. distribution
recnum=recnum + 1,
i=i,
totalcount=totalcount,
thisproc=(" ({i}/~{proccount} this process)".format(
i=i, proccount=totalcount //
ntasks) if ntasks > 1 else "")))
recnum += ntasks
# log.critical("other_values={}".format(repr(other_values)))
srchash = nlpdef.hash(text)
progrec = None
if incremental:
progrec = ifconfig.get_progress_record(pkval, pkstr)
if progrec is not None:
if progrec.srchash == srchash:
log.debug("Record previously processed; skipping")
continue
else:
log.debug("Record has changed")
else:
log.debug("Record is new")
for processor in nlpdef.get_processors():
if incremental:
processor.delete_dest_record(ifconfig,
pkval,
pkstr,
commit=incremental)
processor.process(text, other_values)
# Make a note in the progress database that we've processed a
# source record.
if progrec: # modifying an existing record
progrec.whenprocessedutc = nlpdef.get_now()
progrec.srchash = srchash
else: # creating a new record
progrec = NlpRecord(
# Quasi-key fields:
srcdb=ifconfig.get_srcdb(),
srctable=ifconfig.get_srctable(),
srcpkval=pkval,
srcpkstr=pkstr,
srcfield=ifconfig.get_srcfield(),
nlpdef=nlpdef.get_name(),
# Other fields:
srcpkfield=ifconfig.get_srcpkfield(),
whenprocessedutc=nlpdef.get_now(),
srchash=srchash,
)
with MultiTimerContext(timer, TIMING_PROGRESS_DB_ADD):
session.add(progrec)
# In incremental mode, do we commit immediately, because other
# processes may need this table promptly... ?
# force_commit = False # definitely wrong; crashes as below
# force_commit = incremental
force_commit = ntasks > 1
# - A single source record should not be processed by >1 CRATE
# process. So in theory there should be no conflicts.
# - However, databases can lock in various ways. Can we guarantee
# it'll do something sensible?
# - See also
# https://en.wikipedia.org/wiki/Isolation_(database_systems)
# http://skien.cc/blog/2014/02/06/sqlalchemy-and-race-conditions-follow-up/ # noqa
# http://docs.sqlalchemy.org/en/latest/core/connections.html?highlight=execution_options#sqlalchemy.engine.Connection.execution_options # noqa
# - However, empirically, setting this to False gives
# "Transaction (Process ID xx) was deadlocked on lock resources
# with another process and has been chosen as the deadlock
# victim. Rerun the transaction." -- with a SELECT query.
# - SQL Server uses READ COMMITTED as the default isolation level.
# - https://technet.microsoft.com/en-us/library/jj856598(v=sql.110).aspx # noqa
nlpdef.notify_transaction(
session=session,
n_rows=1,
n_bytes=sys.getsizeof(progrec), # approx
force_commit=force_commit)
nlpdef.commit_all()
def delete_where_no_source(nlpdef: NlpDefinition,
ifconfig: InputFieldConfig,
report_every: int = DEFAULT_REPORT_EVERY,
chunksize: int = DEFAULT_CHUNKSIZE) -> None:
"""
Delete destination records where source records no longer exist.
- Can't do this in a single SQL command, since the engine can't necessarily
see both databases.
- Can't use a single temporary table, since the progress database isn't
necessarily the same as any of the destination database(s).
- Can't do this in a multiprocess way, because we're trying to do a
DELETE WHERE NOT IN.
- So we fetch all source PKs (which, by definition, do exist), stash them
keep them in memory, and do a DELETE WHERE NOT IN based on those
specified values (or, if there are no PKs in the source, delete
everything from the destination).
Problems:
- This is IMPERFECT if we have string source PKs and there are hash
collisions (e.g. PKs for records X and Y both hash to the same thing;
record X is deleted; then its processed version might not be).
- With massive tables, we might run out of memory or (much more likely)
SQL parameter slots. -- This is now happening; error looks like:
pyodbc.ProgrammingError: ('The SQL contains 30807 parameter parkers, but
2717783 parameters were supplied', 'HY000')
A better way might be:
- for each table, make a temporary table in the same database
- populate that table with (source PK integer/hash, source PK string) pairs
- delete where pairs don't match -- is that portable SQL?
http://stackoverflow.com/questions/7356108/sql-query-for-deleting-rows-with-not-in-using-2-columns # noqa
- More efficient would be to make one table per destination database.
On the "delete where multiple fields don't match":
- Single field syntax is
DELETE FROM a WHERE a1 NOT IN (SELECT b1 FROM b)
- Multiple field syntax is
DELETE FROM a WHERE NOT EXISTS (
SELECT 1 FROM b
WHERE a.a1 = b.b1
AND a.a2 = b.b2
)
- In SQLAlchemy, exists():
http://stackoverflow.com/questions/14600619
http://docs.sqlalchemy.org/en/latest/core/selectable.html
- Furthermore, in SQL NULL = NULL is false, and NULL <> NULL is also false,
so we have to do an explicit null check.
You do that with "field == None" (disable
See http://stackoverflow.com/questions/21668606
We're aiming, therefore, for:
DELETE FROM a WHERE NOT EXISTS (
SELECT 1 FROM b
WHERE a.a1 = b.b1
AND (
a.a2 = b.b2
OR (a.a2 IS NULL AND b.b2 IS NULL)
)
)
"""
# -------------------------------------------------------------------------
# Sub-functions
# -------------------------------------------------------------------------
def insert(records_):
n_rows = len(records_)
log.debug("... inserting {} records".format(n_rows))
for db in databases:
session_ = db['session']
temptable_ = db['temptable'] # type: Table
session_.execute(temptable_.insert(), records_)
nlpdef.notify_transaction(session_,
n_rows=n_rows,
n_bytes=sys.getsizeof(records_))
def commit():
for db in databases:
nlpdef.commit(db['session'])
# -------------------------------------------------------------------------
# Main code
# -------------------------------------------------------------------------
# Use info log level, otherwise it looks like our code hangs with very
# large databases.
log.info("delete_where_no_source: examining source table {}.{}; "
"MAY BE SLOW".format(ifconfig.get_srcdb(),
ifconfig.get_srctable()))
# Start our list with the progress database
databases = [{
'session': nlpdef.get_progdb_session(),
'engine': nlpdef.get_progdb_engine(),
'metadata': nlpdef.get_progdb_metadata(),
'temptable': None, # type: Table
}]
# Add the processors' destination databases
for processor in nlpdef.get_processors(): # of type BaseNlpParser
session = processor.get_session()
if any(x['session'] == session for x in databases):
continue # already exists
databases.append({
'session': session,
'engine': processor.get_engine(),
'metadata': processor.get_metadata(),
})
# Make a temporary table in each database (note: the Table objects become
# affiliated to their engine, I think, so make separate ones for each).
log.info("... using {n} destination database(s)".format(n=len(databases)))
log.info("... dropping (if exists) and creating temporary table(s)")
for database in databases:
engine = database['engine']
temptable = Table(
nlpdef.get_temporary_tablename(),
database['metadata'],
Column(FN_SRCPKVAL, BigInteger), # not PK, as may be a hash
Column(FN_SRCPKSTR, String(MAX_STRING_PK_LENGTH)),
**TABLE_KWARGS)
temptable.drop(engine, checkfirst=True)
temptable.create(engine, checkfirst=True)
database['temptable'] = temptable
# Insert PKs into temporary tables
n = count_star(ifconfig.get_source_session(), ifconfig.get_srctable())
log.info("... populating temporary table(s): {} records to go; working in "
"chunks of {}".format(n, chunksize))
i = 0
records = [] # type: List[Dict[str, Any]]
for pkval, pkstr in ifconfig.gen_src_pks():
i += 1
if report_every and i % report_every == 0:
log.info("... src row# {} / {}".format(i, n))
records.append({FN_SRCPKVAL: pkval, FN_SRCPKSTR: pkstr})
if i % chunksize == 0:
insert(records)
records = [] # type: List[Dict[str, Any]]
if records: # remainder
insert(records)
# Commit
commit()
# Index, for speed
log.info("... creating index(es) on temporary table(s)")
for database in databases:
temptable = database['temptable'] # type: Table
index = Index('_temptable_idx', temptable.columns[FN_SRCPKVAL])
index.create(database['engine'])
# DELETE FROM desttable WHERE destpk NOT IN (SELECT srcpk FROM temptable)
log.info("... deleting from progress/destination DBs where appropriate")
# Delete from progress database
prog_db = databases[0]
prog_temptable = prog_db['temptable']
ifconfig.delete_progress_records_where_srcpk_not(prog_temptable)
# Delete from others
for processor in nlpdef.get_processors():
database = [
x for x in databases if x['session'] == processor.get_session()
][0]
temptable = database['temptable']
processor.delete_where_srcpk_not(ifconfig, temptable)
# Drop temporary tables
log.info("... dropping temporary table(s)")
for database in databases:
database['temptable'].drop(database['engine'], checkfirst=True)
# Commit
commit()
def __init__(self, nlpdef: NlpDefinition, cfg_input_name: str) -> None:
"""
Read config from a configparser section, and also associate with a
specific NLP definition.
Args:
nlpdef:
:class:`crate_anon.nlp_manager.nlp_definition.NlpDefinition`,
the master NLP definition, referring to the master config file
etc.
cfg_input_name:
config section name for the input field definition
"""
self.name = cfg_input_name
cfg = nlpdef.get_config_section(
full_sectionname(NlpConfigPrefixes.INPUT, cfg_input_name))
self._nlpdef = nlpdef
self._srcdb = cfg.opt_str(InputFieldConfigKeys.SRCDB)
self._srctable = cfg.opt_str(InputFieldConfigKeys.SRCTABLE)
self._srcpkfield = cfg.opt_str(InputFieldConfigKeys.SRCPKFIELD)
self._srcfield = cfg.opt_str(InputFieldConfigKeys.SRCFIELD)
self._srcdatetimefield = cfg.opt_str(
InputFieldConfigKeys.SRCDATETIMEFIELD, required=False)
# ... new in v0.18.52
# Make these case-sensitive to avoid our failure in renaming SQLA
# Column objects to be lower-case:
self._copyfields = cfg.opt_multiline(
InputFieldConfigKeys.COPYFIELDS) # fieldnames
self._indexed_copyfields = cfg.opt_multiline(
InputFieldConfigKeys.INDEXED_COPYFIELDS)
self._debug_row_limit = cfg.opt_int(
InputFieldConfigKeys.DEBUG_ROW_LIMIT, default=0)
# self._fetch_sorted = opt_bool('fetch_sorted', default=True)
ensure_valid_table_name(self._srctable)
ensure_valid_field_name(self._srcpkfield)
ensure_valid_field_name(self._srcfield)
if self._srcdatetimefield:
ensure_valid_field_name(self._srcdatetimefield)
if len(set(self._indexed_copyfields)) != len(self._indexed_copyfields):
raise ValueError(
f"Redundant indexed_copyfields: {self._indexed_copyfields}")
if len(set(self._copyfields)) != len(self._copyfields):
raise ValueError(f"Redundant copyfields: {self._copyfields}")
indexed_not_copied = set(self._indexed_copyfields) - set(
self._copyfields)
if indexed_not_copied:
raise ValueError(f"Fields in index_copyfields but not in "
f"copyfields: {indexed_not_copied}")
# allfields = [self._srcpkfield, self._srcfield] + self._copyfields
# if len(allfields) != len(set(allfields)):
# raise ValueError(
# f"Field overlap in InputFieldConfig: {section}")
# RE-THOUGHT: OK to copy source text fields etc. if desired.
# It's fine in SQL to say SELECT a, a FROM mytable;
self._db = nlpdef.get_database(self._srcdb)