def filter_noun_chunks(chunks,
exclude_pronouns=False,
exclude_query=None,
exact_match=False):
if exclude_pronouns:
chunks = [
np for np in chunks
if (np.lemma_ != '-PRON-' and not all(tok.pos_ == 'PRON'
for tok in np))
]
if exclude_query is not None:
excl_txt = [exclude_query.lower()]
filtered_chunks = []
for chunk in chunks:
lower_chunk = chunk.text.lower()
found = False
for excl in excl_txt:
if ((not exact_match and
(lower_chunk in excl or excl in lower_chunk))
or lower_chunk == excl):
found = True
break
if not found:
filtered_chunks.append(chunk)
chunks = filtered_chunks
return chunks
def device(self):
assert all(
[dls.device == self.multi_dls[0].device for dls in self.multi_dls])
return self.multi_dls[0].device
def __init__(self, multi_dls, cycle=False):
assert all([dls.device == multi_dls[0].device for dls in multi_dls])
self.multi_dls = multi_dls
self.train = MultiTaskDataloader([dls.train for dls in self.multi_dls],
cycle=cycle)