def find_court_by_key_column(
self, phrase: LineOrPhrase, phrase_finder: PhraseFinder,
column: str) -> Tuple[MatchFound, List[PhraseMatch]]:
found_substrings = phrase_finder.find_word(phrase.text, True)
if len(found_substrings) == 0:
return None
subset = self.courts.loc[self.courts[column] == found_substrings[0][0]]
if len(subset) == 0:
return None
match = MatchFound(subset, phrase.start + found_substrings[0][1],
phrase.start + found_substrings[0][2])
return (match, found_substrings)
def test_abbreviation(self):
text = "In C.D. Ill. we should find"
finder = PhraseFinder(['C.D. Ill.'])
rst = finder.find_word(text, True)
self.assertEqual(1, len(rst))
finder = PhraseFinder(['C.D. Ill.', 'sh', 'should', 'find'])
rst = finder.find_word(text, True)
self.assertEqual(3, len(rst))
class CourtCitationsParser:
"""
https://docs.google.com/spreadsheets/d/1_-Hnr46s8JmTYIFSkcI01gbwqwjnVDoAV1dUv3Bz5fM/edit#gid=0
could be one of the following:
(lalala; DATE; vom) or (lalala; DATE; Beschluss)
or § lalala Court Name (abbreviation)
or
"""
reg_cite_chunk = re.compile(r"\([^0-9\)]+[0-9\.]{2,}[^\)]+\)", re.UNICODE)
reg_trigger_words = re.compile('Beschluss|vom', re.UNICODE | re.IGNORECASE)
reg_token_end = re.compile(r"[;,]")
registries = {
'BStBl': 'Bundessteuerblatt (sonstige gebräuchliche Verwendung)',
'BFH': 'Deutschland, Rechtswesen: Bundesfinanzhof',
'BFHE': 'Sammlung der Entscheidungen des BFH',
'GmS-OGB': 'Beschluss des Gemeinsamen Senats der obersten Gerichtshöfe des Bundes',
'BVerwGE': 'Entscheidungen des Bundesverwaltungsgerichts',
'GrS': 'Beschluss des Großen Senats des BFH',
'BFH-Urteile': 'Bundesfinanzhof Urteile',
'BFH-Beschlüsse': 'Bundesfinanzhof Beschlüsse',
'DstR': 'Deutsches Steuerrecht - DStR',
'KStG':'Körperschaftsteuergesetz'
}
registry_finder = None
reg_split_by_registry = None
# region STATICINIT
if not registry_finder:
registry_finder = PhraseFinder(list(registries.keys()))
if not reg_split_by_registry:
reg_split_by_registry = re.compile("|".join(list(registries.keys())))
# endregion
def __init__(self):
self.locale = 'de'
self.items = [] # List[CourtCitationAnnotation]
self.locale = None
def parse(self, text: str, locale: str = None) -> List[CourtCitationAnnotation]:
self.items = []
self.locale = locale
self.find_citations_in_embraced_text(text)
return self.items
def find_citations_in_embraced_text(self, text: str) -> None:
fragment_start = 0
for embraced_text in CourtCitationsParser.reg_cite_chunk.finditer(text):
start = embraced_text.start()
# process text before braces
fragment = text[fragment_start:start]
self.split_chunk_and_find_citations(fragment, fragment_start)
fragment_start = embraced_text.end() + 1
# process text in braces
self.process_chunks_in_embraced_text(embraced_text, start)
fragment = text[fragment_start:-1]
self.split_chunk_and_find_citations(fragment, fragment_start)
def process_chunks_in_embraced_text(self, embraced_text: str, start) -> None:
parts = embraced_text.group().split(';')
for part in parts:
self.get_detail_from_chunk(part, start)
start += len(part) + 1
return
def split_chunk_and_find_citations(self, text: str, start: int) -> None:
chunks = self.split_text_by_keywords(text)
for chunk in chunks:
self.get_detail_from_chunk(chunk[0], chunk[1] + start)
def get_detail_from_chunk(self, chunk_text: str, chunk_start: int) -> None:
chunk_body = chunk_text.strip(r'() \t')
dates = self.get_dates_from_text(chunk_body)
registries = self.get_registries_from_text(chunk_body)
triggers = CourtCitationsParser.reg_trigger_words.search(chunk_body)
if not triggers and len(registries) == 0 and len(dates) == 0:
return
start = chunk_start + chunk_text.find(chunk_body)
end = start + len(chunk_body)
ant = CourtCitationAnnotation(name=chunk_body,
coords=(start, end),
text=chunk_body,
locale=self.locale)
ant.locale = self.locale
if len(registries) > 0:
ant.name = CourtCitationsParser.registries[registries[0].value]
ant.short_name = self.get_reference_from_registry(registries[0], chunk_body)
self.items.append(ant)
def get_reference_from_registry(self, registry: PossibleToken,
chunk_body: str) -> str:
start = registry.coords[0]
end = -1
end_match = CourtCitationsParser.reg_token_end.search(chunk_body[start:])
if end_match:
end = end_match.start()
return chunk_body[start: end + 1].strip(' \t.,;()')
def get_registries_from_text(self, text: str) -> List[PossibleToken]:
reg_names = [(m, 100) for m in CourtCitationsParser.registry_finder.find_word(text, ignore_case=False)]
# if the case is not the same, the probability is 50%
reg_names += [(m, 50) for m in CourtCitationsParser.registry_finder.find_word(text, ignore_case=True)]
reg_names.sort(key=lambda n: n[0][1] - n[1] * 1000)
toks = []
for match_prob in reg_names:
tok = PossibleToken('registry', match_prob[0][0],
(match_prob[0][1], match_prob[0][2]),
match_prob[1])
toks.append(tok)
return toks
def get_dates_from_text(self, text: str) -> List[PossibleToken]:
try:
date_ents = list(get_dates(text))
except TypeError:
date_ents = []
date_ents.sort(key=lambda d: d['location_start'])
tokens = []
for d in date_ents:
tok = PossibleToken('date', d['value'],
(d['location_start'], d['location_end']), 100)
tokens.append(tok)
if len(tokens) > 0:
return tokens
# try get years only
for year in year_parser.year_parser.get_years_with_coords_from_string(text):
tokens.append(PossibleToken('date', str(year[0]),
(year[1], year[2]), 50))
return tokens
def split_text_by_keywords(self, text: str) -> List[Tuple[str, int]]:
matches = list(CourtCitationsParser.reg_split_by_registry.finditer(text))
chunks = []
for i in range(len(matches)):
match = matches[i]
ending = -1
if i < len(matches) - 1:
ending = matches[i + 1].start() - 1
phrase_break = CourtCitationsParser.reg_token_end.search(text[match.end():ending])
if phrase_break:
ending = min(ending, phrase_break.start())
chunks.append((text[match.start():ending], match.start()))
return chunks
def __init__(self, ptrs: ParserInitParams):
"""
:param ptrs.court_pattern_checker: a regex or None, the parser skips the phrase if pattern doesn't match the phrase
:param ptrs.column_names['type']: "Court Type", e.g. 'Federal District Court'
:param ptrs.column_names['name']: "Court Name", e.g. 'Southern Georgia District Court'
:param ptrs.column_names['jurisdiction']: "Jurisdiction", e.g. 'Federal'
:param ptrs.column_names['alias']: "Alias", e.g. 'C.D. Cal'
:param ptrs.dataframe_paths: like ['data/us_courts.csv', ...]
:param ptrs.split_ptrs: phrase splitting processor parameters, see LineProcessor class
:param ptrs.key_word_preproc_func: a function used to pre-process column values used in text search
dataframe_paths is a collection of *.CSV files that contain the data like:
| Jurisdiction || Court Type || Court Name || ... |
| Federal || Verfassungsgericht || Bundesverfassungsgericht || ... |
The column 'Court Name' (you may provide another column name instead of Court Name
in param: court_name_column) should contain unique values that precisely identify each
of the court given.
The columns 'Court Type' (param: court_type_column) and 'Jurisdiction'
(param: jurisdiction_column) in couple may or may not precisely identify the court given.
At least this parser can identify the court's type and return the annotation that
neither specifies the court's name nor jurisdiction
The court_pattern_checker parameter speeds up the parsing process:
- the whole text or the line would be skipped if this line doesn't match the court_pattern_checker
E.g., you can pass re.compile('court', re.IGNORECASE) for searching courts' annotations
for the En locale
The split_ptrs specify how the parser splits the text into phrases.
Each phrase can contain zero ore one court annotations. See LineProcessor class.
For a courts parser phrase bounds usually include punctuation (.,;!?) and conjunctions
(and, or) or (und, oder)
The example function for key_word_preproc_func is:
def preproc_func(text):
return re.sub('e$', '[e]?', text)
"""
self.phrase_match_pattern = None if ptrs.court_pattern_checker is None \
else ptrs.court_pattern_checker
self.court_type_column = ptrs.column_names['type']
self.court_name_column = ptrs.column_names['name']
self.court_alias_column = ptrs.column_names['alias']
self.jurisdiction_column = ptrs.column_names['jurisdiction']
self.proc = LineProcessor()
self.phrase_split_ptrs = ptrs.split_ptrs
self.annotations = []
self.courts = None
self.load_courts(ptrs.dataframe_paths)
# unique columns
self.finder_court_name = PhraseFinder(
UniversalCourtsParser.get_unique_col_values(
self.courts[self.court_name_column]),
ptrs.key_word_preproc_func)
self.finder_court_alias = None if len(self.court_alias_column) == 0 else \
PhraseFinder(UniversalCourtsParser.get_unique_col_values(
self.courts[self.court_alias_column]), ptrs.key_word_preproc_func)
# non-unique columns
self.finder_court_type = PhraseFinder(
UniversalCourtsParser.get_unique_col_values(
self.courts[self.court_type_column]),
ptrs.key_word_preproc_func)
self.finder_jur = PhraseFinder(
UniversalCourtsParser.get_unique_col_values(
self.courts[self.jurisdiction_column]),
ptrs.key_word_preproc_func)
class UniversalCourtsParser:
"""
The class describes a "constructor" for building locale (and region) specific
parsers, that find reference to courts within the text.
Use the parse() method to find all reference to courts from the
text provided.
Each reference is a dictionary with two keys:
- "attrs" key leads to the "coordinates" (starting and ending characters) of the
occurrence within the provided text
- "tags" key leads to another dictionary, which contains:
- court official name
- court's jurisdiction ...
In order to parse the text you are supposed to create your locale (or region) specific instance of
UniversalCourtsParser. See the constructor below:
"""
def __init__(self, ptrs: ParserInitParams):
"""
:param ptrs.court_pattern_checker: a regex or None, the parser skips the phrase if pattern doesn't match the phrase
:param ptrs.column_names['type']: "Court Type", e.g. 'Federal District Court'
:param ptrs.column_names['name']: "Court Name", e.g. 'Southern Georgia District Court'
:param ptrs.column_names['jurisdiction']: "Jurisdiction", e.g. 'Federal'
:param ptrs.column_names['alias']: "Alias", e.g. 'C.D. Cal'
:param ptrs.dataframe_paths: like ['data/us_courts.csv', ...]
:param ptrs.split_ptrs: phrase splitting processor parameters, see LineProcessor class
:param ptrs.key_word_preproc_func: a function used to pre-process column values used in text search
dataframe_paths is a collection of *.CSV files that contain the data like:
| Jurisdiction || Court Type || Court Name || ... |
| Federal || Verfassungsgericht || Bundesverfassungsgericht || ... |
The column 'Court Name' (you may provide another column name instead of Court Name
in param: court_name_column) should contain unique values that precisely identify each
of the court given.
The columns 'Court Type' (param: court_type_column) and 'Jurisdiction'
(param: jurisdiction_column) in couple may or may not precisely identify the court given.
At least this parser can identify the court's type and return the annotation that
neither specifies the court's name nor jurisdiction
The court_pattern_checker parameter speeds up the parsing process:
- the whole text or the line would be skipped if this line doesn't match the court_pattern_checker
E.g., you can pass re.compile('court', re.IGNORECASE) for searching courts' annotations
for the En locale
The split_ptrs specify how the parser splits the text into phrases.
Each phrase can contain zero ore one court annotations. See LineProcessor class.
For a courts parser phrase bounds usually include punctuation (.,;!?) and conjunctions
(and, or) or (und, oder)
The example function for key_word_preproc_func is:
def preproc_func(text):
return re.sub('e$', '[e]?', text)
"""
self.phrase_match_pattern = None if ptrs.court_pattern_checker is None \
else ptrs.court_pattern_checker
self.court_type_column = ptrs.column_names['type']
self.court_name_column = ptrs.column_names['name']
self.court_alias_column = ptrs.column_names['alias']
self.jurisdiction_column = ptrs.column_names['jurisdiction']
self.proc = LineProcessor()
self.phrase_split_ptrs = ptrs.split_ptrs
self.annotations = []
self.courts = None
self.load_courts(ptrs.dataframe_paths)
# unique columns
self.finder_court_name = PhraseFinder(
UniversalCourtsParser.get_unique_col_values(
self.courts[self.court_name_column]),
ptrs.key_word_preproc_func)
self.finder_court_alias = None if len(self.court_alias_column) == 0 else \
PhraseFinder(UniversalCourtsParser.get_unique_col_values(
self.courts[self.court_alias_column]), ptrs.key_word_preproc_func)
# non-unique columns
self.finder_court_type = PhraseFinder(
UniversalCourtsParser.get_unique_col_values(
self.courts[self.court_type_column]),
ptrs.key_word_preproc_func)
self.finder_jur = PhraseFinder(
UniversalCourtsParser.get_unique_col_values(
self.courts[self.jurisdiction_column]),
ptrs.key_word_preproc_func)
def parse(self, text: str) -> List[dict]:
"""
:param text: the text being processed
:return: annotations - List[dict]
Here is an example of the method's call:
ret = processor.parse("Bei dir läuft, deine Verfassungsgerichtshof des Freistaates Sachsen rauchen Joints vor der Kamera")
ret[0]['attrs'] = {'start': 14, 'end': 97}
ret[0]['tags'] = {'Extracted Entity Type': 'court',
'Extracted Entity Court Name': 'Verfassungsgerichtshof des Freistaates Sachsen',
'Extracted Entity Court Type': 'Verfassungsgericht',
'Extracted Entity Court Jurisdiction': 'Sachsen'}
"""
self.annotations = []
self.find_courts_by_alias_in_whole_text(text)
# if the whole text doesn't contain the key word (gericht) - skip all the following
if self.phrase_match_pattern is not None:
if self.phrase_match_pattern.search(text, re.IGNORECASE) is None:
return self.annotations
for phrase in self.proc.split_text_on_line_with_endings(
text, self.phrase_split_ptrs):
# if the phrase doesn't contain the key word (e.g., gericht for deutsche) - skip the phrase
if self.phrase_match_pattern is not None:
if self.phrase_match_pattern.search(phrase.text,
re.IGNORECASE) is None:
continue
self.find_court_by_any_key(phrase)
return self.annotations
def load_courts(self, dataframe_paths: List[str]):
frames = []
dtypes = {
self.court_type_column: str,
self.court_name_column: str,
self.jurisdiction_column: str
}
if self.court_alias_column:
dtypes[self.court_alias_column] = str
for path in dataframe_paths:
frame = pd.read_csv(path,
encoding="utf-8",
error_bad_lines=False,
converters=dtypes)
frames.append(frame)
self.courts = pd.concat(frames)
def find_courts_by_alias_in_whole_text(self, text: str) -> None:
if self.finder_court_alias is None:
return
for m in self.finder_court_alias.find_word(text):
alias = m[0]
rows = self.courts.loc[self.courts[self.court_alias_column] ==
alias]
match_found = MatchFound(rows, m[1], m[2])
self.add_annotation(match_found)
def find_court_by_any_key(self, phrase: LineOrPhrase):
# find by court names
matches = []
matches += self.find_court_by_name(phrase)
matches += self.find_court_by_type_and_jurisdiction(phrase)
matches = [m for m in matches if m is not None]
if len(matches) == 0:
return
# find the best match
matches.sort(key=lambda m: m.make_sort_key())
self.add_annotation(matches[0])
def find_court_by_name(self, phrase: LineOrPhrase) -> List[MatchFound]:
match = self.find_court_by_key_column(phrase, self.finder_court_name,
self.court_name_column)
if match is None:
return []
match[0].court_name = match[1][0][0]
return [match[0]]
def find_court_by_key_column(
self, phrase: LineOrPhrase, phrase_finder: PhraseFinder,
column: str) -> Tuple[MatchFound, List[PhraseMatch]]:
found_substrings = phrase_finder.find_word(phrase.text, True)
if len(found_substrings) == 0:
return None
subset = self.courts.loc[self.courts[column] == found_substrings[0][0]]
if len(subset) == 0:
return None
match = MatchFound(subset, phrase.start + found_substrings[0][1],
phrase.start + found_substrings[0][2])
return (match, found_substrings)
def find_court_by_type_and_jurisdiction(
self, phrase: LineOrPhrase) -> List[MatchFound]:
court_types = self.finder_court_type.find_word(phrase.text, True)
if len(court_types) == 0:
return []
court_jurs = self.finder_jur.find_word(phrase.text, True)
if len(court_types) != 1 or len(court_jurs) > 1:
# special case: 2 ore more courts within the same phrase
# (without commas or conjuctions)
matches = []
for ct in court_types:
m = MatchFound([], phrase.start + ct[1], phrase.start + ct[2])
m.court_type = ct[0]
m.court_name = ct[0]
matches.append(m)
return matches
if len(court_jurs) == 0:
subset = self.courts.loc[self.courts[self.court_type_column] ==
court_types[0][0]]
else:
subset = self.courts.loc[
(self.courts[self.court_type_column] == court_types[0][0])
& (self.courts[self.jurisdiction_column] == court_jurs[0][0])]
match = MatchFound(subset, phrase.start,
phrase.start + court_types[0][2])
if len(subset) != 1:
match.court_name = court_types[0][0]
match.court_type = court_types[0][0]
return [match]
def add_annotation(self, match: MatchFound):
mlen = len(match.subset)
name = match.subset[self.court_name_column].values[0] \
if match.is_exact else \
match.court_name if match.court_name is not None else \
match.subset[self.court_name_column].values[0] if mlen > 0 else ''
court_type = match.subset[self.court_type_column].values[0] \
if match.is_exact else \
match.court_type if match.court_type is not None else \
match.subset[self.court_type_column].values[0] if mlen > 0 else ''
jurisdiction = match.subset[self.jurisdiction_column].values[0] \
if match.is_exact else \
match.jurisdiction if match.jurisdiction is not None else \
match.subset[self.jurisdiction_column].values[0] if mlen > 0 else ''
ant = dict(attrs={
'start': match.entry_start,
'end': match.entry_end
},
tags={
'Extracted Entity Type': 'court',
'Extracted Entity Court Name': name,
'Extracted Entity Court Type': court_type,
'Extracted Entity Court Jurisdiction': jurisdiction
})
self.annotations.append(ant)
@staticmethod
def get_unique_col_values(col_values):
return [c for c in col_values.unique() if c]