def write_labels(self, labels: Union[List[dict], List[Label]], index: Optional[str] = None):
index = index or self.label_index
label_objects = [Label.from_dict(l) if isinstance(l, dict) else l for l in labels]
for label in label_objects:
label_id = uuid.uuid4()
self.indexes[index][label_id] = label
def write_labels(self,
labels: Union[List[Label], List[dict]],
index: Optional[str] = None):
index = index or self.label_index
if index and not self.client.indices.exists(index=index):
self._create_label_index(index)
# Make sure we comply to Label class format
label_objects = [
Label.from_dict(l) if isinstance(l, dict) else l for l in labels
]
labels_to_index = []
for label in label_objects:
_label = {
"_op_type":
"index" if self.update_existing_documents else "create",
"_index": index,
**label.to_dict()
} # type: Dict[str, Any]
labels_to_index.append(_label)
bulk(self.client,
labels_to_index,
request_timeout=300,
refresh="wait_for")
def get_all_labels(
self,
index: Optional[str] = None,
filters: Optional[Dict[str, List[str]]] = None) -> List[Label]:
index = index or self.label_index
result = self.get_all_documents_in_index(index=index, filters=filters)
labels = [Label.from_dict(hit["_source"]) for hit in result]
return labels
def write_labels(self, labels, index=None):
labels = [
Label.from_dict(l) if isinstance(l, dict) else l for l in labels
]
index = index or self.label_index
for label in labels:
label_orm = LabelORM(
document_id=label.document_id,
no_answer=label.no_answer,
origin=label.origin,
question=label.question,
is_correct_answer=label.is_correct_answer,
is_correct_document=label.is_correct_document,
answer=label.answer,
offset_start_in_doc=label.offset_start_in_doc,
model_id=label.model_id,
index=index,
)
self.session.add(label_orm)
self.session.commit()
def get_all_labels(self,
index: str = "label",
filters: Optional[dict] = None) -> List[Label]:
result = self.get_all_documents_in_index(index=index, filters=filters)
labels = [Label.from_dict(hit["_source"]) for hit in result]
return labels