def test_translate_equivalence(data: Dict[str, Any], translate_params_equiv: str, compare_output: bool): """ Tests whether the output and scores generated by sockeye.translate with translate_params_equiv are equal to the previously generated outputs, referenced in the data dictionary. """ out_path = os.path.join(data['work_dir'], "test.out.equiv") params = "{} {} {}".format( sockeye.translate.__file__, TRANSLATE_PARAMS_COMMON.format(model=data['model'], input=data['test_source'], output=out_path), translate_params_equiv) if 'test_source_factors' in data: params += TRANSLATE_WITH_FACTORS_COMMON.format( input_factors=" ".join(data['test_source_factors'])) with patch.object(sys, "argv", params.split()): sockeye.translate.main() # Collect translate outputs and scores translate_outputs_equiv = collect_translate_output_and_scores(out_path) assert 'test_outputs' in data assert len(data['test_outputs']) == len(translate_outputs_equiv) if compare_output: for json_output, json_output_equiv in zip(data['test_outputs'], translate_outputs_equiv): assert json_output['translation'] == json_output_equiv[ 'translation'] assert abs(json_output['score'] - json_output_equiv['score']) < 0.01 or \ np.isnan(json_output['score'] - json_output_equiv['score'])
def test_constrained_decoding_against_ref(data: Dict[str, Any], translate_params: str): constrained_inputs = create_reference_constraints(data['test_inputs'], data['test_outputs']) new_test_source_path = os.path.join(data['work_dir'], "test_constrained.txt") with open(new_test_source_path, 'w') as out: for json_line in constrained_inputs: print(json_line, file=out) out_path_constrained = os.path.join(data['work_dir'], "out_constrained.txt") params = "{} {} {} --json-input --output-type translation_with_score --beam-size 1 --batch-size 1 --nbest-size 1".format( sockeye.translate.__file__, TRANSLATE_PARAMS_COMMON.format(model=data['model'], input=new_test_source_path, output=out_path_constrained), translate_params) with patch.object(sys, "argv", params.split()): sockeye.translate.main() constrained_outputs = collect_translate_output_and_scores( out_path_constrained) assert len(constrained_outputs) == len( data['test_outputs']) == len(constrained_inputs) for json_input, json_constrained, json_unconstrained in zip( constrained_inputs, constrained_outputs, data['test_outputs']): # Make sure the constrained output is the same as we got when decoding unconstrained assert json_constrained['translation'] == json_unconstrained[ 'translation'] data['test_constrained_inputs'] = constrained_inputs data['test_constrained_outputs'] = constrained_outputs return data
def _test_constrained_type(constraint_type: str, data: Dict[str, Any], translate_params: str): constrained_inputs = _create_constrained_inputs(constraint_type, data['test_inputs'], data['test_outputs']) new_test_source_path = os.path.join(data['work_dir'], "test_constrained.txt") with open(new_test_source_path, 'w') as out: for json_line in constrained_inputs: print(json_line, file=out) out_path_constrained = os.path.join(data['work_dir'], "out_constrained.txt") params = "{} {} {} --json-input --output-type translation_with_score".format( sockeye.translate.__file__, TRANSLATE_PARAMS_COMMON.format(model=data['model'], input=new_test_source_path, output=out_path_constrained), translate_params) with patch.object(sys, "argv", params.split()): sockeye.translate.main() constrained_outputs = collect_translate_output_and_scores( out_path_constrained) assert len(constrained_outputs) == len( data['test_outputs']) == len(constrained_inputs) for json_source, json_constrained, json_unconstrained in zip( constrained_inputs, constrained_outputs, data['test_outputs']): jobj = json.loads(json_source) if jobj.get(constraint_type) is None: # if there were no constraints, make sure the output is the same as the unconstrained output assert json_constrained['translation'] == json_unconstrained[ 'json_constrained'] else: restriction = jobj[constraint_type][0] if constraint_type == 'constraints': # for positive constraints, ensure the constraint is in the constrained output assert restriction in json_constrained['translation'] else: # for negative constraints, ensure the constraints is *not* in the constrained output assert restriction not in json_constrained['translation']
def test_translate_equivalence(data: Dict[str, Any], translate_params_equiv: str, compare_output: bool): """ Tests whether the output and scores generated by sockeye.translate with translate_params_equiv are equal to the previously generated outputs, referenced in the data dictionary. """ out_path = os.path.join(data['work_dir'], "test.out.equiv") out_with_target_prefix_path = os.path.join( data['work_dir'], "test_with_target_prefix.out.equiv") # First set of params (with target prefix in JSON format) params = "{} {} {}".format( sockeye.translate.__file__, TRANSLATE_PARAMS_COMMON.format( model=data['model'], input=data['test_source_with_target_prefix'], output=out_with_target_prefix_path), translate_params_equiv) params += TRANSLATE_WITH_JSON_FORMAT with patch.object(sys, "argv", params.split()): sockeye.translate.main() # Collect translate outputs and scores translate_outputs_with_target_prefix_equiv = collect_translate_output_and_scores( out_with_target_prefix_path) # Second set of params (without using target prefix) params = "{} {} {}".format( sockeye.translate.__file__, TRANSLATE_PARAMS_COMMON.format(model=data['model'], input=data['test_source'], output=out_path), translate_params_equiv) if 'test_source_factors' in data: params += TRANSLATE_WITH_FACTORS_COMMON.format( input_factors=" ".join(data['test_source_factors'])) with patch.object(sys, "argv", params.split()): sockeye.translate.main() # Collect translate outputs and scores translate_outputs_equiv = collect_translate_output_and_scores(out_path) assert 'test_outputs' in data assert 'test_with_target_prefix_outputs' in data assert len(data['test_outputs']) == len( data['test_with_target_prefix_outputs']) == len( translate_outputs_with_target_prefix_equiv) == len( translate_outputs_equiv) if compare_output: for json_output, json_output_with_target_prefix, json_output_equiv, json_output_with_target_prefix_equiv in zip( data['test_outputs'], data['test_with_target_prefix_outputs'], translate_outputs_equiv, translate_outputs_with_target_prefix_equiv): assert json_output['translation'] == json_output_equiv['translation'], \ f"'{json_output['translation']}' vs. '{json_output_equiv['translation']}'" assert json_output_with_target_prefix['translation'] == json_output_with_target_prefix_equiv['translation'], \ f"'{json_output_with_target_prefix['translation']}' vs. '{json_output_with_target_prefix_equiv['translation']}'" assert abs(json_output['score'] - json_output_equiv['score']) < 0.01 or \ np.isnan(json_output['score'] - json_output_equiv['score']), \ f"'{json_output['score']}' vs. '{ json_output_equiv['score']}'" assert abs(json_output_with_target_prefix['score'] - json_output_with_target_prefix_equiv['score']) < 0.01 or \ np.isnan(json_output_with_target_prefix['score'] - json_output_with_target_prefix_equiv['score']), \ f"'{json_output_with_target_prefix['score']}' vs. '{ json_output_with_target_prefix_equiv['score']}'" # Check translation output always includes target prefix tokens prefix = json_output_with_target_prefix['target_prefix'].split() translation = json_output_with_target_prefix['translation'].split() ending = min(len(prefix), len(translation)) assert prefix[:ending] == translation[:ending], \ f"'{prefix[:ending]}' vs. '{translation[:ending]}'" # Check translation output factors always include target prefix factors if 'target_prefix_factors' in json_output_with_target_prefix: prefix = json_output_with_target_prefix[ 'target_prefix_factors'] if len(prefix) > 0: for j in range(1, len(prefix) + 1): factors_from_translation = json_output_with_target_prefix[ f'factor{j}'] ending = min(len(prefix[j - 1]), len(factors_from_translation)) assert prefix[j - 1][:ending] == factors_from_translation[:ending], \ f"'{prefix[j - 1][:ending]}' vs. '{factors_from_translation[:ending]}' from . '{json_output_with_target_prefix}'"