def test_paths_to_blocks_with_missing(self, test_data_001, test_data_002, make_target_series, splitter_func, test_data_no_file): no_file_ds = pd.Series([''], index=[test_data_no_file.uid]) target_ds = pd.concat([ make_target_series(test_data_001), make_target_series(test_data_002), no_file_ds ], axis=0) input = pd.Series([ test_data_001.pdf, test_data_002.pdf, test_data_no_file.pdf, ], index=[ test_data_001.uid, test_data_002.uid, test_data_no_file.uid, ]) output = PDFDecoder.paths_to_blocks(input, split_func=splitter_func, missing_file='ignore') assert output.equals(target_ds) output = PDFDecoder.threaded_paths_to_blocks(input, split_func=splitter_func, missing_file='ignore') assert output.equals(target_ds) with pytest.raises(FileNotFoundError): PDFDecoder.threaded_paths_to_blocks( input, split_func=splitter_func, missing_file='raise', )
def test_invalid_return_type(self): text_series = pd.Series( ['first text\n\n2lines', 'second text\n\nline\n\nlineagain']) with pytest.raises(ValueError): PDFDecoder.threaded_texts_to_blocks( text_series, return_type='incorrect_input', ) with pytest.raises(ValueError): PDFDecoder.text_to_blocks_series( text_series.iloc[0], return_type='incorrect_input', )
def test_path_to_blocks_series(self, test_data_001, test_data_002, make_target_series, splitter_func): target = make_target_series(test_data_001) blocks_series = (PDFDecoder.path_to_blocks_series( test_data_001.pdf, split_func=splitter_func, index=test_data_001.uid)) assert target.equals(blocks_series) target = make_target_series(test_data_002) blocks_series = (PDFDecoder.path_to_blocks_series( test_data_002.pdf, split_func=splitter_func, index=test_data_002.uid)) assert target.equals(blocks_series)
def test_empty_content_raise( self, test_data_001, test_data_002, make_target_series, splitter_func, ): content_list = [] for test_data in [test_data_001, test_data_002]: with open(test_data.pdf, mode='rb') as content: content_list.append(content.read()) content_list[0] = None content_ds = pd.Series(content_list, index=[test_data_001.uid, test_data_002.uid]) # Case where error should be raised (content input is None) with pytest.raises(RuntimeError): PDFDecoder.threaded_contents_to_text(content_ds, none_content='raise')
def test_texts_to_blocks_as_list(self): text_series = pd.Series( ['first text\n\n2lines', 'second text\n\nline\n\nlineagain'], index=pd.Index(['001', '002']), ) blocks_ds = (PDFDecoder.threaded_texts_to_blocks( text_series, return_type='as_list', )) target = pd.Series( [['first text', '2lines'], ['second text', 'line', 'lineagain']], index=pd.Index(['001', '002'])) assert blocks_ds.equals(target) blocks_ds = PDFDecoder.text_to_blocks_series('first text\n\n2lines', index='001', return_type='as_list') print(blocks_ds) print(target.loc['001']) assert blocks_ds.loc['001'] == target.loc['001']
def test_paths_to_blocks(self, test_data_001, test_data_002, make_target_series, splitter_func): target_ds = pd.concat([ make_target_series(test_data_001), make_target_series(test_data_002) ], axis=0) input = pd.Series([test_data_001.pdf, test_data_002.pdf], index=[test_data_001.uid, test_data_002.uid]) output = PDFDecoder.paths_to_blocks(input, split_func=splitter_func) assert output.equals(target_ds)
def test_content_to_text_to_blocks( self, test_data_001, test_data_002, make_target_series, splitter_func, ): target_ds = pd.concat([ make_target_series(test_data_001), make_target_series(test_data_002) ], axis=0) content_list = [] for test_data in [test_data_001, test_data_002]: with open(test_data.pdf, mode='rb') as content: content_list.append(content.read()) content_ds = pd.Series(content_list, index=[test_data_001.uid, test_data_002.uid]) text_ds = PDFDecoder.threaded_contents_to_text(content_ds) blocks_ds = (PDFDecoder.threaded_texts_to_blocks( text_ds, split_func=splitter_func)) assert blocks_ds.equals(target_ds)
def index(): if request.method == 'POST': session.clear() attached = request.files['attached'] text = PDFDecoder.content_to_text(attached) splitted = pd.Series([splitter(text)]) result = simselector.predict(splitted).iloc[0] session['result'] = result attached.stream.seek(0) session['filecontent'] = attached.read() return redirect(url_for('home.result')) else: return render_template('home/index.html')
def test_path_to_blocks(self, test_data_001, splitter_func): target = pd.read_csv( test_data_001.blocks, encoding='utf-8-sig', sep=';', header=None, names=['blocks'], squeeze=True, ) target = list(target) target.append('\x0c') blocks = PDFDecoder.path_to_blocks(test_data_001.pdf, split_func=splitter_func) assert target == blocks
def test_invalid_arg( self, test_data_001, test_data_002, make_target_series, splitter_func, ): content_list = [] for test_data in [test_data_001, test_data_002]: with open(test_data.pdf, mode='rb') as content: content_list.append(content.read()) content_ds = pd.Series(content_list, index=[test_data_001.uid, test_data_002.uid]) # Unexpected none_content argument with pytest.raises(ValueError): (PDFDecoder.threaded_contents_to_text( content_ds, none_content='incorrect_input'))
def test_empty_content_empty( self, test_data_001, test_data_002, make_target_series, splitter_func, ): # Passing case where content input is empty string content_list = [] for test_data in [test_data_001, test_data_002]: with open(test_data.pdf, mode='rb') as content: content_list.append(content.read()) content_list[0] = b'' content_ds = pd.Series(content_list, index=[test_data_001.uid, test_data_002.uid]) texts_ds = (PDFDecoder.threaded_contents_to_text( content_ds, none_content='to_empty')) assert texts_ds.iloc[0] == '' assert (texts_ds.iloc[1] == ( Path(test_data_002.txt).read_text(encoding='utf-8-sig') + '\x0c'))
def test_path_to_text_incorrect_params(self): with pytest.raises(ValueError): PDFDecoder.path_to_text('', missing_file='incorrect')
def test_path_to_text_corrupted(self): path = Path(__file__).parent / 'test_data' / '003_corrupted_pdf.pdf' assert PDFDecoder.path_to_text(path, missing_file='raise') == ''
def test_path_to_text_no_file(self): assert PDFDecoder.path_to_text('incorect/path', missing_file='ignore') == '' with pytest.raises(FileNotFoundError): PDFDecoder.path_to_text('incorect/path')
def test_path_to_text_2(self, test_data_002): target = Path(test_data_002.txt).read_text(encoding='utf-8-sig') target = target + '\x0c' assert PDFDecoder.path_to_text(test_data_002.pdf) == target
def test_path_to_text(self, test_data_001): target = Path(test_data_001.txt).read_text() target += '\x0c' assert PDFDecoder.path_to_text(test_data_001.pdf) == target
def test_incorrect_content_type(self): with pytest.raises(AttributeError): PDFDecoder.content_to_text(b'', none_content='to_empty')
def test_none_content(self): content = BytesIO(b'') with pytest.raises(RuntimeError): PDFDecoder.content_to_text(content, none_content='raise') assert (PDFDecoder.content_to_text(content, none_content='to_empty') == '')
def test_invalid_none_content_arg(self): with pytest.raises(ValueError): PDFDecoder.content_to_text(b'', none_content='incorrect_input')