def process_task_stream(sentence): from ufal.udpipe import Model, Pipeline model_path = MODELS_DIR + MODEL_NAMES[ 'russian'] # language harcoded so far model = Model.load(model_path) pipeline = Pipeline(model, '', '', '', '') print('...loaded the model') return process_data(sentence, pipeline)
def process_task(file_ids_list=None): from ufal.udpipe import Model, Pipeline model_path = MODELS_DIR + MODEL_NAMES[ 'russian'] # language harcoded so far model = Model.load(model_path) pipeline = Pipeline(model, '', '', '', '') file_to_process = boilerplate.get_file( file_ids_list[0]) # getting the content of the file print('...loaded the model') # parsed = pipeline.process(file_to_process.decode('utf-8')) # print('...parsed the sentence') # print(parsed) return process_data(file_to_process, pipeline)
def test_process_data_bad_values(): with pytest.raises(AttributeError): assert [contents for _, contents in process_data({"test": 1})]
def test_process_data(input_data, expected_result): processed_data = [contents for _, contents in process_data(input_data)] assert processed_data == expected_result