def test_encode_decode_json():
test_obj = [
'foo', {
'bar': 'baz'
}, ['xyz', 'zyx'], 'moo', 'ąčęėįšųūž', 42, 3.14, True, False, None
]
expected_json = '["foo",{"bar":"baz"},["xyz","zyx"],"moo","ąčęėįšųūž",42,3.14,true,false,null]'
encoded_json = encode_json(json_obj=test_obj, pretty=False)
assert encoded_json == expected_json
decoded_json = decode_json(json_string=encoded_json)
assert decoded_json == test_obj
# Encoding errors
with pytest.raises(McEncodeJSONException):
# noinspection PyTypeChecker
encode_json(None)
with pytest.raises(McEncodeJSONException):
# noinspection PyTypeChecker
encode_json("strings can't be encoded")
with pytest.raises(McDecodeJSONException):
# noinspection PyTypeChecker
decode_json(None)
with pytest.raises(McDecodeJSONException):
# noinspection PyTypeChecker
decode_json('not JSON')
def _request_for_text(self, text: str) -> Request:
text = decode_object_from_bytes_if_needed(text)
# CLIFF annotator URL
config = py_get_config()
url = config.get('nytlabels', {}).get('annotator_url', None)
if url is None:
raise McNYTLabelsAnnotatorException(
"Unable to determine NYTLabels annotator URL to use.")
# Create JSON request
log.debug("Converting text to JSON request...")
try:
text_json = encode_json({'text': text})
except Exception as ex:
# Not critical, might happen to some stories, no need to shut down the annotator
raise McNYTLabelsAnnotatorException(
"Unable to encode text to a JSON request: %(exception)s\nText: %(text)s"
% {
'exception': str(ex),
'text': text,
})
log.debug("Done converting text to JSON request.")
request = Request(method='POST', url=url)
request.set_content_type('application/json; charset=utf-8')
request.set_content(text_json)
return request
def annotate_and_store_for_story(self, db: DatabaseHandler, stories_id: int) -> None:
"""Run the annotation for the story, store results in key-value store."""
if not self.annotator_is_enabled():
fatal_error("Annotator is not enabled in the configuration.")
# MC_REWRITE_TO_PYTHON: remove after rewrite to Python
if isinstance(stories_id, bytes):
stories_id = decode_object_from_bytes_if_needed(stories_id)
stories_id = int(stories_id)
if self.story_is_annotated(db=db, stories_id=stories_id):
log.warning("Story %d is already annotated, so I will overwrite it." % stories_id)
if not self.story_is_annotatable(db=db, stories_id=stories_id):
log.warning("Story %d is not annotatable." % stories_id)
return
story_sentences = db.query("""
SELECT story_sentences_id, sentence_number, sentence
FROM story_sentences
WHERE stories_id = %(stories_id)s
ORDER BY sentence_number
""", {'stories_id': stories_id}).hashes()
if story_sentences is None:
raise McJSONAnnotatorException("Unable to fetch story sentences for story %s." % stories_id)
# MC_REWRITE_TO_PYTHON: remove after rewrite to Perl
if isinstance(story_sentences, dict):
story_sentences = [story_sentences]
log.info("Annotating story's %d concatenated sentences..." % stories_id)
sentences_concat_text = ' '.join(s['sentence'] for s in story_sentences)
annotation = self.__annotate_text(sentences_concat_text)
if annotation is None:
raise McJSONAnnotatorException(
"Unable to annotate story sentences concatenation for story %d." % stories_id)
json_annotation = None
try:
json_annotation = encode_json(annotation)
if json_annotation is None:
raise McJSONAnnotatorException("JSON annotation is None for annotation %s." % str(annotation))
except Exception as ex:
fatal_error("Unable to encode annotation to JSON: %s\nAnnotation: %s" % (str(ex), str(annotation)))
log.info("Done annotating story's %d concatenated sentences." % stories_id)
log.debug("JSON length: %d" % len(json_annotation))
log.info("Storing annotation results for story %d..." % stories_id)
try:
self.__postgresql_store.store_content(db=db, object_id=stories_id, content=json_annotation.encode('utf-8'))
except Exception as ex:
fatal_error("Unable to store annotation result: %s\nJSON annotation: %s" % (str(ex), json_annotation))
log.info("Done storing annotation results for story %d." % stories_id)
def __nyt_labels_sample_response(_: HashServer.Request) -> Union[str, bytes]:
"""Mock annotator."""
response = ""
response += "HTTP/1.0 200 OK\r\n"
response += "Content-Type: application/json; charset=UTF-8\r\n"
response += "\r\n"
response += encode_json(self.__sample_nyt_labels_response())
return response