def main():
parser = argparse.ArgumentParser(description="""Computes rationale and final class classification scores""", formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument('--data_dir', dest='data_dir', required=True, help='Which directory contains a {train,val,test}.jsonl file?')
parser.add_argument('--split', dest='split', required=True, help='Which of {train,val,test} are we scoring on?')
parser.add_argument('--strict', dest='strict', required=False, action='store_true', default=False, help='Do we perform strict scoring?')
parser.add_argument('--results', dest='results', required=True, help="""Results File
Contents are expected to be jsonl of:
{
"annotation_id": str, required
# these classifications *must not* overlap
# these classifications *must not* overlap
"rationales": List[
{
"docid": str, required
"hard_rationale_predictions": List[{
"start_token": int, inclusive, required
"end_token": int, exclusive, required
}], optional,
# token level classifications, a value must be provided per-token
# in an ideal world, these correspond to the hard-decoding above.
"soft_rationale_predictions": List[float], optional.
# sentence level classifications, a value must be provided for every
# sentence in each document, or not at all
"soft_sentence_predictions": List[float], optional.
}
],
# the classification the model made for the overall classification task
"classification": str, optional
# A probability distribution output by the model. We require this to be normalized.
"classification_scores": Dict[str, float], optional
# The next two fields are measures for how faithful your model is (the
# rationales it predicts are in some sense causal of the prediction), and
# how sufficient they are. We approximate a measure for comprehensiveness by
# asking that you remove the top k%% of tokens from your documents,
# running your models again, and reporting the score distribution in the
# "comprehensiveness_classification_scores" field.
# We approximate a measure of sufficiency by asking exactly the converse
# - that you provide model distributions on the removed k%% tokens.
# 'k' is determined by human rationales, and is documented in our paper.
# You should determine which of these tokens to remove based on some kind
# of information about your model: gradient based, attention based, other
# interpretability measures, etc.
# scores per class having removed k%% of the data, where k is determined by human comprehensive rationales
"comprehensiveness_classification_scores": Dict[str, float], optional
# scores per class having access to only k%% of the data, where k is determined by human comprehensive rationales
"sufficiency_classification_scores": Dict[str, float], optional
# the number of tokens required to flip the prediction - see "Is Attention Interpretable" by Serrano and Smith.
"tokens_to_flip": int, optional
}
When providing one of the optional fields, it must be provided for *every* instance.
The classification, classification_score, and comprehensiveness_classification_scores
must together be present for every instance or absent for every instance.
""")
parser.add_argument('--iou_thresholds', dest='iou_thresholds', required=False, nargs='+', type=float, default=[0.5], help='''Thresholds for IOU scoring.
These are used for "soft" or partial match scoring of rationale spans.
A span is considered a match if the size of the intersection of the prediction
and the annotation, divided by the union of the two spans, is larger than
the IOU threshold. This score can be computed for arbitrary thresholds.
''')
parser.add_argument('--score_file', dest='score_file', required=False, default=None, help='Where to write results?')
args = parser.parse_args()
results = load_jsonl(args.results)
docids = set(chain.from_iterable([rat['docid'] for rat in res['rationales']] for res in results))
docs = load_flattened_documents(args.data_dir, docids)
verify_instances(results, docs)
# load truth
annotations = annotations_from_jsonl(os.path.join(args.data_dir, args.split + '.jsonl'))
docids |= set(chain.from_iterable((ev.docid for ev in chain.from_iterable(ann.evidences)) for ann in annotations))
has_final_predictions = _has_classifications(results)
scores = dict()
if args.strict:
if not args.iou_thresholds:
raise ValueError("--iou_thresholds must be provided when running strict scoring")
if not has_final_predictions:
raise ValueError("We must have a 'classification', 'classification_score', and 'comprehensiveness_classification_score' field in order to perform scoring!")
# TODO think about offering a sentence level version of these scores.
if _has_hard_predictions(results):
truth = list(chain.from_iterable(Rationale.from_annotation(ann) for ann in annotations))
pred = list(chain.from_iterable(Rationale.from_instance(inst) for inst in results))
if args.iou_thresholds is not None:
iou_scores = partial_match_score(truth, pred, args.iou_thresholds)
scores['iou_scores'] = iou_scores
# NER style scoring
rationale_level_prf = score_hard_rationale_predictions(truth, pred)
scores['rationale_prf'] = rationale_level_prf
token_level_truth = list(chain.from_iterable(rat.to_token_level() for rat in truth))
token_level_pred = list(chain.from_iterable(rat.to_token_level() for rat in pred))
token_level_prf = score_hard_rationale_predictions(token_level_truth, token_level_pred)
scores['token_prf'] = token_level_prf
else:
logging.info("No hard predictions detected, skipping rationale scoring")
if _has_soft_predictions(results):
flattened_documents = load_flattened_documents(args.data_dir, docids)
paired_scoring = PositionScoredDocument.from_results(results, annotations, flattened_documents, use_tokens=True)
token_scores = score_soft_tokens(paired_scoring)
scores['token_soft_metrics'] = token_scores
else:
logging.info("No soft predictions detected, skipping rationale scoring")
if _has_soft_sentence_predictions(results):
documents = load_documents(args.data_dir, docids)
paired_scoring = PositionScoredDocument.from_results(results, annotations, documents, use_tokens=False)
sentence_scores = score_soft_tokens(paired_scoring)
scores['sentence_soft_metrics'] = sentence_scores
else:
logging.info("No sentence level predictions detected, skipping sentence-level diagnostic")
if has_final_predictions:
flattened_documents = load_flattened_documents(args.data_dir, docids)
class_results = score_classifications(results, annotations, flattened_documents)
scores['classification_scores'] = class_results
else:
logging.info("No classification scores detected, skipping classification")
pprint.pprint(scores)
if args.score_file:
with open(args.score_file, 'w') as of:
json.dump(scores, of, indent=4, sort_keys=True)
def main():
parser = argparse.ArgumentParser(
description=""" """, formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument(
'--data_dir',
dest='data_dir',
required=True,
help='Which directory contains a {train,val,test}.jsonl file?')
parser.add_argument(
'--output_dir',
dest='output_dir',
required=True,
help='Where shall we write intermediate models + final data to?')
parser.add_argument(
'--model_params',
dest='model_params',
required=True,
help=
'JSoN file for loading arbitrary model parameters (e.g. optimizers, pre-saved files, etc.'
)
args = parser.parse_args()
BATCH_FIRST = True
with open(args.model_params, 'r') as fp:
logging.debug(f'Loading model parameters from {args.model_params}')
model_params = json.load(fp)
train, val, test = load_datasets(args.data_dir)
documents = load_documents(args.data_dir)
document_vocab = set(
chain.from_iterable(chain.from_iterable(documents.values())))
annotation_vocab = set(
chain.from_iterable(e.query.split() for e in chain(train, val, test)))
logging.debug(
f'Loaded {len(documents)} documents with {len(document_vocab)} unique words'
)
# this ignores the case where annotations don't align perfectly with token boundaries, but this isn't that important
vocab = document_vocab | annotation_vocab
unk_token = 'UNK'
classifier, word_interner, de_interner, evidence_classes = initialize_model(
model_params, vocab, batch_first=BATCH_FIRST, unk_token=unk_token)
classifier = classifier.cuda()
logging.debug(
f'Including annotations, we have {len(vocab)} total words in the data, with embeddings for {len(word_interner)}'
)
interned_documents = intern_documents(documents, word_interner, unk_token)
interned_train = intern_annotations(train, word_interner, unk_token)
interned_val = intern_annotations(val, word_interner, unk_token)
interned_test = intern_annotations(test, word_interner, unk_token)
classifier, results = train_classifier(classifier, args.output_dir,
interned_train, interned_val,
interned_documents, model_params,
evidence_classes)
val_rats, test_rats = decode(
classifier,
interned_train,
interned_val,
interned_test,
interned_documents,
evidence_classes,
batch_size=model_params['classifier']['batch_size'],
tensorize_model_inputs=True,
threshold=model_params['classifier']['threshold'],
k_fraction=model_params['classifier']['k_fraction'],
has_query=bool(model_params['classifier']['has_query']))
with open(os.path.join(args.output_dir, 'val_decoded.jsonl'),
'w') as val_output:
for line in val_rats:
val_output.write(json.dumps(line))
val_output.write('\n')
with open(os.path.join(args.output_dir, 'test_decoded.jsonl'),
'w') as test_output:
for line in test_rats:
test_output.write(json.dumps(line))
test_output.write('\n')
def main():
parser = argparse.ArgumentParser(
description="""Trains a pipeline model.
Step 1 is evidence identification, that is identify if a given sentence is evidence or not
Step 2 is evidence classification, that is given an evidence sentence, classify the final outcome for the final task (e.g. sentiment or significance).
These models should be separated into two separate steps, but at the moment:
* prep data (load, intern documents, load json)
* convert data for evidence identification - in the case of training data we take all the positives and sample some negatives
* side note: this sampling is *somewhat* configurable and is done on a per-batch/epoch basis in order to gain a broader sampling of negative values.
* train evidence identification
* convert data for evidence classification - take all rationales + decisions and use this as input
* train evidence classification
* decode first the evidence, then run classification for each split
""",
formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument(
'--data_dir',
dest='data_dir',
required=True,
help='Which directory contains a {train,val,test}.jsonl file?')
parser.add_argument(
'--output_dir',
dest='output_dir',
required=True,
help='Where shall we write intermediate models + final data to?')
parser.add_argument(
'--model_params',
dest='model_params',
required=True,
help=
'JSoN file for loading arbitrary model parameters (e.g. optimizers, pre-saved files, etc.'
)
args = parser.parse_args()
BATCH_FIRST = True
assert BATCH_FIRST
with open(args.model_params, 'r') as fp:
logger.info(f'Loading model parameters from {args.model_params}')
model_params = json.load(fp)
logger.info(
f'Params: {json.dumps(model_params, indent=2, sort_keys=True)}')
train, val, test = load_datasets(args.data_dir)
docids = set(e.docid for e in chain.from_iterable(
chain.from_iterable(
map(lambda ann: ann.evidences, chain(train, val, test)))))
documents = load_documents(args.data_dir, docids)
logger.info(f'Loaded {len(documents)} documents')
# this ignores the case where annotations don't align perfectly with token boundaries, but this isn't that important
unk_token = '<unk>'
evidence_identifier, evidence_classifier, word_interner, de_interner, evidence_classes, tokenizer = initialize_models(
model_params, batch_first=BATCH_FIRST, unk_token=unk_token)
logger.info(f'We have {len(word_interner)} wordpieces')
logger.info(f'Interning documents')
interned_documents = {
d: bert_intern_doc(bert_tokenize_doc(doc, tokenizer), tokenizer)
for d, doc in documents.items()
}
interned_train = bert_intern_annotation(train, tokenizer)
interned_val = bert_intern_annotation(val, tokenizer)
interned_test = bert_intern_annotation(test, tokenizer)
# train the evidence identifier
logger.info('Beginning training of the evidence identifier')
evidence_identifier = evidence_identifier.cuda()
optimizer = None
scheduler = None
evidence_identifier, evidence_ident_results = train_evidence_identifier(
evidence_identifier,
args.output_dir,
interned_train,
interned_val,
interned_documents,
model_params,
optimizer=optimizer,
scheduler=scheduler,
tensorize_model_inputs=True)
evidence_identifier = evidence_identifier.cpu(
) # free GPU space for next model
# train the evidence classifier
logger.info('Beginning training of the evidence classifier')
evidence_classifier = evidence_classifier.cuda()
optimizer = None
scheduler = None
evidence_classifier, evidence_class_results = train_evidence_classifier(
evidence_classifier,
args.output_dir,
interned_train,
interned_val,
interned_documents,
model_params,
class_interner=evidence_classes,
tensorize_model_inputs=True)
# decode
logger.info('Beginning final decoding')
evidence_identifier = evidence_identifier.cuda()
pipeline_batch_size = min([
model_params['evidence_classifier']['batch_size'],
model_params['evidence_identifier']['batch_size']
])
pipeline_results, train_decoded, val_decoded, test_decoded = decode(
evidence_identifier,
evidence_classifier,
interned_train,
interned_val,
interned_test,
interned_documents,
evidence_classes,
pipeline_batch_size,
BATCH_FIRST,
decoding_docs=documents)
write_jsonl(train_decoded,
os.path.join(args.output_dir, 'train_decoded.jsonl'))
write_jsonl(val_decoded, os.path.join(args.output_dir,
'val_decoded.jsonl'))
write_jsonl(test_decoded,
os.path.join(args.output_dir, 'test_decoded.jsonl'))
with open(os.path.join(args.output_dir, 'identifier_results.json'), 'w') as ident_output, \
open(os.path.join(args.output_dir, 'classifier_results.json'), 'w') as class_output:
ident_output.write(json.dumps(evidence_ident_results))
class_output.write(json.dumps(evidence_class_results))
for k, v in pipeline_results.items():
if type(v) is dict:
for k1, v1 in v.items():
logging.info(f'Pipeline results for {k}, {k1}={v1}')
else:
logging.info(f'Pipeline results {k}\t={v}')
def main():
parser = argparse.ArgumentParser(
description="""Trains a pipeline model.
Step 1 is evidence identification, that is identify if a given sentence is evidence or not
Step 2 is evidence classification, that is given an evidence sentence, classify the final outcome for the final task (e.g. sentiment or significance).
These models should be separated into two separate steps, but at the moment:
* prep data (load, intern documents, load json)
* convert data for evidence identification - in the case of training data we take all the positives and sample some negatives
* side note: this sampling is *somewhat* configurable and is done on a per-batch/epoch basis in order to gain a broader sampling of negative values.
* train evidence identification
* convert data for evidence classification - take all rationales + decisions and use this as input
* train evidence classification
* decode first the evidence, then run classification for each split
""",
formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument(
'--data_dir',
dest='data_dir',
required=True,
help='Which directory contains a {train,val,test}.jsonl file?')
parser.add_argument(
'--output_dir',
dest='output_dir',
required=True,
help='Where shall we write intermediate models + final data to?')
parser.add_argument(
'--model_params',
dest='model_params',
required=True,
help=
'JSoN file for loading arbitrary model parameters (e.g. optimizers, pre-saved files, etc.'
)
args = parser.parse_args()
BATCH_FIRST = True
with open(args.model_params, 'r') as fp:
logging.debug(f'Loading model parameters from {args.model_params}')
model_params = json.load(fp)
train, val, test = load_datasets(args.data_dir)
docids = set(e.docid for e in chain.from_iterable(
chain.from_iterable(
map(lambda ann: ann.evidences, chain(train, val, test)))))
documents = load_documents(args.data_dir, docids)
document_vocab = set(
chain.from_iterable(chain.from_iterable(documents.values())))
annotation_vocab = set(
chain.from_iterable(e.query.split() for e in chain(train, val, test)))
logging.debug(
f'Loaded {len(documents)} documents with {len(document_vocab)} unique words'
)
# this ignores the case where annotations don't align perfectly with token boundaries, but this isn't that important
vocab = document_vocab | annotation_vocab
unk_token = 'UNK'
evidence_identifier, evidence_classifier, word_interner, de_interner, evidence_classes = initialize_models(
model_params, vocab, batch_first=BATCH_FIRST, unk_token=unk_token)
logging.debug(
f'Including annotations, we have {len(vocab)} total words in the data, with embeddings for {len(word_interner)}'
)
interned_documents = intern_documents(documents, word_interner, unk_token)
interned_train = intern_annotations(train, word_interner, unk_token)
interned_val = intern_annotations(val, word_interner, unk_token)
interned_test = intern_annotations(test, word_interner, unk_token)
assert BATCH_FIRST # for correctness of the split dimension for DataParallel
evidence_identifier, evidence_ident_results = train_evidence_identifier(
evidence_identifier.cuda(),
args.output_dir,
interned_train,
interned_val,
interned_documents,
model_params,
tensorize_model_inputs=True)
evidence_classifier, evidence_class_results = train_evidence_classifier(
evidence_classifier.cuda(),
args.output_dir,
interned_train,
interned_val,
interned_documents,
model_params,
class_interner=evidence_classes,
tensorize_model_inputs=True)
pipeline_batch_size = min([
model_params['evidence_classifier']['batch_size'],
model_params['evidence_identifier']['batch_size']
])
pipeline_results, train_decoded, val_decoded, test_decoded = decode(
evidence_identifier,
evidence_classifier,
interned_train,
interned_val,
interned_test,
interned_documents,
evidence_classes,
pipeline_batch_size,
tensorize_model_inputs=True)
write_jsonl(train_decoded,
os.path.join(args.output_dir, 'train_decoded.jsonl'))
write_jsonl(val_decoded, os.path.join(args.output_dir,
'val_decoded.jsonl'))
write_jsonl(test_decoded,
os.path.join(args.output_dir, 'test_decoded.jsonl'))
with open(os.path.join(args.output_dir, 'identifier_results.json'), 'w') as ident_output, \
open(os.path.join(args.output_dir, 'classifier_results.json'), 'w') as class_output:
ident_output.write(json.dumps(evidence_ident_results))
class_output.write(json.dumps(evidence_class_results))
for k, v in pipeline_results.items():
if type(v) is dict:
for k1, v1 in v.items():
logging.info(f'Pipeline results for {k}, {k1}={v1}')
else:
logging.info(f'Pipeline results {k}\t={v}')
def main():
parser = argparse.ArgumentParser(description="""Trains a pipeline model.
Step 1 is evidence identification, that is identify if a given sentence is evidence or not
Step 2 is evidence classification, that is given an evidence sentence, classify the final outcome for the final task
(e.g. sentiment or significance).
These models should be separated into two separate steps, but at the moment:
* prep data (load, intern documents, load json)
* convert data for evidence identification - in the case of training data we take all the positives and sample some
negatives
* side note: this sampling is *somewhat* configurable and is done on a per-batch/epoch basis in order to gain a
broader sampling of negative values.
* train evidence identification
* convert data for evidence classification - take all rationales + decisions and use this as input
* train evidence classification
* decode first the evidence, then run classification for each split
""", formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument('--data_dir', dest='data_dir', required=True,
help='Which directory contains a {train,val,test}.jsonl file?')
parser.add_argument('--output_dir', dest='output_dir', required=True,
help='Where shall we write intermediate models + final data to?')
parser.add_argument('--model_params', dest='model_params', required=True,
help='JSoN file for loading arbitrary model parameters (e.g. optimizers, pre-saved files, etc.')
args = parser.parse_args()
assert BATCH_FIRST
os.makedirs(args.output_dir, exist_ok=True)
with open(args.model_params, 'r') as fp:
logger.info(f'Loading model parameters from {args.model_params}')
model_params = json.load(fp)
logger.info(f'Params: {json.dumps(model_params, indent=2, sort_keys=True)}')
train, val, test = load_datasets(args.data_dir)
docids = set(e.docid for e in
chain.from_iterable(chain.from_iterable(map(lambda ann: ann.evidences, chain(train, val, test)))))
documents = load_documents(args.data_dir, docids)
logger.info(f'Loaded {len(documents)} documents')
# this ignores the case where annotations don't align perfectly with token boundaries, but this isn't that important
unk_token = '<unk>'
evidence_identifier, evidence_token_identifier, evidence_classifier, word_interner, de_interner, evidence_classes, tokenizer = \
initialize_models(model_params,
batch_first=BATCH_FIRST,
unk_token=unk_token)
if not ((evidence_identifier is None) ^ (evidence_token_identifier is None)):
raise ValueError('Exactly one of the evidence identifier and evidence token identifier must be defined, not both!')
logger.info(f'We have {len(word_interner)} wordpieces')
cache = os.path.join(args.output_dir, 'preprocessed.pkl')
if os.path.exists(cache):
logger.info(f'Loading interned documents from {cache}')
(interned_documents, interned_document_token_slices) = torch.load(cache)
else:
logger.info(f'Interning documents')
special_token_map = {
'SEP': [evidence_classifier.sep_token_id],
'[SEP]': [evidence_classifier.sep_token_id],
'[sep]': [evidence_classifier.sep_token_id],
'UNK': [tokenizer.unk_token_id],
'[UNK]': [tokenizer.unk_token_id],
'[unk]': [tokenizer.unk_token_id],
'PAD': [tokenizer.unk_token_id],
'[PAD]': [tokenizer.unk_token_id],
'[pad]': [tokenizer.unk_token_id],
}
interned_documents = {}
interned_document_token_slices = {}
for d, doc in documents.items():
tokenized, w_slices = bert_tokenize_doc(doc, tokenizer, special_token_map=special_token_map)
interned_documents[d] = bert_intern_doc(tokenized, tokenizer, special_token_map=special_token_map)
interned_document_token_slices[d] = w_slices
torch.save((interned_documents, interned_document_token_slices), cache)
interned_train = bert_intern_annotation(train, tokenizer)
interned_val = bert_intern_annotation(val, tokenizer)
interned_test = bert_intern_annotation(test, tokenizer)
# train the evidence identifier
if evidence_identifier is not None:
logger.info('Beginning training of the evidence sentence identifier')
evidence_identifier = evidence_identifier.cuda()
optimizer = None
scheduler = None
evidence_identifier, evidence_ident_results = train_evidence_identifier(evidence_identifier,
args.output_dir,
interned_train,
interned_val,
interned_documents,
model_params,
optimizer=optimizer,
scheduler=scheduler,
tensorize_model_inputs=True)
evidence_identifier = evidence_identifier.cpu() # free GPU space for next model
else:
logger.info('No evidence sentence identifier provided; skipping')
evidence_identifier, evidence_ident_results = None, None
if evidence_token_identifier is not None:
logger.info('Beginning training of the evidence token identifier')
evidence_token_identifier = evidence_token_identifier.cuda()
evidence_token_identifier, evidence_token_identifier_results = \
train_evidence_token_identifier(evidence_token_identifier,
args.output_dir,
interned_train,
interned_val,
interned_documents=interned_documents,
source_documents=documents,
token_mapping=interned_document_token_slices,
model_pars=model_params,
tensorize_model_inputs=True
)
evidence_token_identifier = evidence_token_identifier.cpu()
else:
logger.info('No evidence token identifier provided; skipping')
evidence_token_identifier_results = None
# train the evidence classifier
logger.info('Beginning training of the evidence classifier')
evidence_classifier = evidence_classifier.cuda()
optimizer = None
scheduler = None
evidence_classifier, evidence_class_results = train_evidence_classifier(evidence_classifier,
args.output_dir,
interned_train,
interned_val,
interned_documents,
model_params,
optimizer=optimizer,
scheduler=scheduler,
class_interner=evidence_classes,
tensorize_model_inputs=True,
token_only_evidence=(evidence_token_identifier is not None))
# decode
logger.info('Beginning final decoding')
if evidence_identifier is not None:
pipeline_batch_size = min([model_params['evidence_classifier']['batch_size'],
model_params['evidence_identifier']['batch_size']])
pipeline_results, train_decoded, val_decoded, test_decoded = decode(evidence_identifier.cuda(),
evidence_classifier.cuda(),
interned_train,
interned_val,
interned_test,
interned_documents,
evidence_classes,
pipeline_batch_size,
BATCH_FIRST,
decoding_docs=documents)
elif evidence_token_identifier is not None:
pipeline_batch_size = min([model_params['evidence_classifier']['batch_size'],
model_params['evidence_token_identifier']['batch_size']])
use_cose_hack = bool(model_params['evidence_token_identifier'].get('cose_data_hack', 0))
if use_cose_hack:
logger.info("Using COS-E data prep and processing hacks!")
logger.info("These hacks impact evidence identification, classification, and decoding. They are not general")
pipeline_results, train_decoded, val_decoded, test_decoded = decode_evidence_tokens_and_classify(evidence_token_identifier.cuda(),
evidence_classifier.cuda(),
interned_train,
interned_val,
interned_test,
interned_documents,
documents,
interned_document_token_slices, # token_mapping
evidence_classes,
pipeline_batch_size,
decoding_docs=documents,
use_cose_hack=use_cose_hack)
else:
raise StateError('Impossible state!')
write_jsonl(train_decoded, os.path.join(args.output_dir, 'train_decoded.jsonl'))
write_jsonl(val_decoded, os.path.join(args.output_dir, 'val_decoded.jsonl'))
write_jsonl(test_decoded, os.path.join(args.output_dir, 'test_decoded.jsonl'))
with open(os.path.join(args.output_dir, 'identifier_results.json'), 'w') as ident_output, \
open(os.path.join(args.output_dir, 'classifier_results.json'), 'w') as class_output:
ident_output.write(json.dumps(evidence_ident_results))
class_output.write(json.dumps(evidence_class_results))
for k, v in pipeline_results.items():
if type(v) is dict:
for k1, v1 in v.items():
logging.info(f'Pipeline results for {k}, {k1}={v1}')
else:
logging.info(f'Pipeline results {k}\t={v}')
