def test_get_as_pd(self):
detector = DocumentFieldMultilineRegexDetector()
detector.csv_content = self.csv_text
df = detector.get_as_pandas_df()
self.assertIsNotNone(df)
self.assertEqual((
6,
2,
), df.shape)
def build_detectors(
cls, detector: DocumentFieldMultilineRegexDetector,
log_function: Callable) -> List[CsvRegexpsDetectionRow]:
try:
df = detector.get_as_pandas_df()
except Exception as e:
cls.log_error(
log_function,
f'CSV data is corrupted for field {detector.field_id}', e)
return []
if df.shape[0] == 0:
return []
if df.shape[1] != 2:
cls.log_error(
log_function, 'CSV data is has wrong number ' +
f'of columns ({df.shape[1]}) for field {detector.field_id}')
return []
detectors = [] # type: List[CsvRegexpsDetectionRow]
for i, row in df.iterrows():
try:
detected_value = row[0]
detector_reg_raw = row[1]
if detector.regexps_pre_process_lower:
reg_pattern = re.compile(detector_reg_raw, re.IGNORECASE)
else:
reg_pattern = re.compile(detector_reg_raw)
detectors.append(
CsvRegexpsDetectionRow(reg_pattern, detected_value,
detector))
except Exception as e:
cls.log_error(
log_function, 'CSV data is corrupted for field ' +
f'{detector.field_id} at line #{i}', e)
return detectors
def test_combine_dfs(self):
# first row is the same, second row - same value, another pattern
# third row - same pattern, another value, last row - brand new
another_text = """
,value,pattern
0,"Big Bank & Company (004578) (Knight, Bobby (Charlotte); Bryant, Koby (Charlotte); Williams, Gary (Charlotte); Johnson, Magic (Charlotte); Lobo, Rebecca (Charlotte))","\bbig\s{1,5}bank\s{1,5}.{1,5}\s{1,5}company\s{1,5}(004578)\b"
1,"Family Name Limited (173437) (Tanner, Rebecca (Houston); Saget, Bob (Houston))","\bfamily\s{1,5}guy(173437)\b"
2,"Eye-Eyes Communications (018951)","\ball\s{1,5}eyes\s{1,5}communications\s{1,5}(018951)\b"
3,"John Smith Archives, LLC d/b/a Charlie (085292) (Flay, Bobby (New York))","\bcharlie\s{1,5}(085292)\b"
"""
with StringIO(another_text) as cs_stream:
df = pd.read_csv(cs_stream, usecols=[1, 2])
detector = DocumentFieldMultilineRegexDetector()
detector.csv_content = self.csv_text
detector.update_checksum()
detector.combine_with_dataframe(df)
df_new = detector.get_as_pandas_df()
row_val = [] # type: List[Tuple[str, str]]
for i, row in df_new.iterrows():
row_val.append((
row[0],
row[1],
))
self.assertEqual(8, len(row_val))
self.assertTrue((
'John Smith Archives, LLC d/b/a Charlie (085292) (Flay, Bobby (New York))',
'\bcharlie\s{1,5}(085292)\b',
) in row_val)
self.assertTrue((
'Big Bank & Company (004578) (Knight, Bobby (Charlotte); Bryant, Koby '
+
'(Charlotte); Williams, Gary (Charlotte); Johnson, Magic (Charlotte); '
+ 'Lobo, Rebecca (Charlotte))',
'\bbig\s{1,5}bank\s{1,5}.{1,5}\s{1,5}company\s{1,5}(004578)\b',
) in row_val)
self.assertTrue((
'Family Name Limited (173437) (Tanner, Rebecca (Houston); Saget, Bob (Houston))',
'\bfamily\s{1,5}guy(173437)\b',
) in row_val)
self.assertTrue((
'Family Name Limited (173437) (Tanner, Rebecca (Houston); Saget, Bob (Houston))',
'\bfamily\s{1,5}name(173437)\b',
) in row_val)
self.assertTrue((
'Eye-Eyes Communications (018951)',
'\ball\s{1,5}eyes\s{1,5}communications\s{1,5}(018951)\b',
) in row_val)
# this one is replaced
self.assertFalse((
'All Eyes Communications (018951) (Moore, Michael (New York); Tarantino, Quentin '
+
'(San Francisco); Lee, Spike (New York); Levinson, Barry (Charlotte))',
'\ball\s{1,5}eyes\s{1,5}communications\s{1,5}(018951)\b',
) in row_val)
def test_checksum(self):
detector = DocumentFieldMultilineRegexDetector()
detector.csv_content = self.csv_text
detector.update_checksum()
self.assertGreater(len(detector.csv_checksum), 10)
cs_old = detector.csv_checksum
detector.csv_content = detector.csv_content[:-1] + ';'
detector.update_checksum()
self.assertGreater(len(detector.csv_checksum), 10)
self.assertNotEqual(cs_old, detector.csv_checksum)
def save_detector_settings_csv(
self, detectors_by_value: Dict[str, List[str]]) -> None:
detector = DocumentFieldMultilineRegexDetector()
detector.document_field = self.document_field
df = pd.DataFrame(columns=['value', 'pattern'])
df.set_index("pattern", inplace=True)
for field_val in detectors_by_value:
for include_reg_value in detectors_by_value[field_val]:
df = df.append(
{
'value': field_val,
'pattern': include_reg_value
},
ignore_index=True)
df.drop_duplicates(subset='pattern', inplace=True)
try:
existing = DocumentFieldMultilineRegexDetector.objects.get(
document_field_id=self.document_field.uid
) # type: DocumentFieldMultilineRegexDetector
except DocumentFieldMultilineRegexDetector.DoesNotExist:
detector.csv_content = df.to_csv()
detector.update_checksum()
detector.save()
return
# just update CSV content and hashsum
if self.drop_previous_field_detectors:
existing.csv_content = df.to_csv()
existing.update_checksum()
existing.save()
return
# join these options with existing one
# overwriting duplicates by detected_value or regexp pattern
existing.combine_with_dataframe(df)
existing.save()
def render(self, name, value, attrs=None, renderer=None):
from apps.document.models import DocumentFieldMultilineRegexDetector
df = DocumentFieldMultilineRegexDetector.get_csv_as_pandas_df(value)
if df is None:
df = pd.DataFrame(columns=['value', 'pattern'])
df.sort_values('value', inplace=True)
# build an HTML table
markup = f'<table name="{name}_table" ' +\
'class="table table-striped table-bordered table-hover table-condensed">' +\
'<tr><th>Value</th><th>Pattern</th></tr>\n'
for i, row in df.iterrows():
cell_value = html.escape(row[0])
cell_pattern = html.escape(row[1])
markup += f'<tr><td>{cell_value}</td><td>{cell_pattern}</td></tr>'
markup += '<tr><td data-row="new-row"><a href="#">Add row...</a></td>' + \
'<td data-row="new-row"><a href="#">Add row...</a></td></tr>'
markup += '</table>\n'
html_value = html.escape(value or '', quote=True)
markup += f'<input name="{name}" type="hidden" value="{html_value}" />\n'
return markup
def setup_mock():
doc_field.uid = 'ABCDEF'
doc_field.code = 'client'
csv_text = """
,value,pattern
0,"Big Bank & Company (004578) (Knight, Bobby (Charlotte); Bryant, Koby (Charlotte); Williams, Gary (Charlotte); Johnson, Magic (Charlotte); Lobo, Rebecca (Charlotte))","\bbig\s{1,5}bank\s{1,5}.{1,5}\s{1,5}company\s{1,5}(004578)\b"
1,"Family Name Limited (173437) (Tanner, Rebecca (Houston); Saget, Bob (Houston))","family\s{1,5}name\s{1,5}\(173437\)"
2,"Financial Services & Co. (015607) (Spelling, Tori (Chicago); Priestley, Jason (Dallas); Perry, Luke (New York); Doherty, Shannon (Chicago); Garth, Jenny (Chicago))","\bfinancial\s{1,5}services\s{1,5}.{1,5}(015607)\b"
3,"Food Wholsale, Inc. (056230) (Jenner, Bruce (Chicago))","\bfood\s{1,5}wholsale,(056230)\b"
4,"All Eyes Communications (018951) (Moore, Michael (New York); Tarantino, Quentin (San Francisco); Lee, Spike (New York); Levinson, Barry (Charlotte))","\ball\s{1,5}eyes\s{1,5}communications\s{1,5}(018951)\b"
5,"Joe Smith Archives, LLC d/b/a Foxtrot (085292) (Flay, Bobby (New York))","\bfoxtrot\s{1,5}(085292)\b
\bjoe\s{1,5}smith\s{1,5}archives\b" """
detector = DocumentFieldMultilineRegexDetector()
detector.csv_content = csv_text
detector.document_field = doc_field
detector.update_checksum()
CsvRegexpsDetectionCacheMock.detector_by_field[doc_field.uid] = detector
def save_detector_settings(cls,
detector: DocumentFieldMultilineRegexDetector):
detector.save()