csvreader = CSVReader(encoding="ISO-8859-1")
jlr = JSONLineReader()
formatter = TextAnnotationFormatter(WaseemLabelSchema(), preprocessing=pp)
formatter2 = TextAnnotationFormatter(WaseemHovyLabelSchema(),
preprocessing=pp,
mapping={
0: 0,
1: 1,
2: 2,
3: 0
})
df = DavidsonFormatter(DavidsonToZLabelSchema(),
preprocessing=pp,
mapping={
0: 0,
1: 1,
2: 2
})
datasets_tr = [
DataSet(file=sexism_file_tr,
reader=jlr,
formatter=formatter,
name=None),
DataSet(file=racism_file_tr,
reader=jlr,
formatter=formatter,
name=None),
DataSet(file=neither_file_tr,
reader=jlr,
racism_file_dv = os.path.join("data", "waseem_r.dv.json")
neither_file_dv = os.path.join("data", "waseem_n.dv.json")
waseem_hovy_dv = os.path.join("data", "amateur_expert.dv.json")
csvreader = CSVReader(encoding="ISO-8859-1")
jlr = JSONLineReader()
formatter = TextAnnotationFormatter(WaseemLabelSchema(), preprocessing=pp)
formatter2 = TextAnnotationFormatter(WaseemHovyLabelSchema(),
preprocessing=pp,
mapping={
0: 0,
1: 1,
2: 2,
3: 0
})
df = DavidsonFormatter(DavidsonLabelSchema(), preprocessing=pp)
datasets_tr = [
DataSet(file=sexism_file_tr,
reader=jlr,
formatter=formatter,
name=None),
DataSet(file=racism_file_tr,
reader=jlr,
formatter=formatter,
name=None),
DataSet(file=neither_file_tr,
reader=jlr,
formatter=formatter,
name=None),
DataSet(file=waseem_hovy_tr,