def parse_format(self, source, binary=None, service=None):
with open(get_fixture_path(source, 'ap')) as fp:
data = json.load(fp)
with self.app.app_context():
with patch.dict(superdesk.resources, resources):
with requests_mock.mock() as mock:
if binary:
with open(get_fixture_path(binary, 'ap'), 'rb') as f:
mock.get(data['data']['item']['renditions']
['preview']['href'],
content=f.read())
parsed = parser.parse(data, self.provider)
parsed['_id'] = 'generated-id'
parsed['unique_id'] = 1
parsed['family_id'] = parsed['_id']
parsed['renditions'] = {
'original': {
'media': 'abcd-media',
'mimetype': 'image/jpeg'
}
}
if service:
parsed.setdefault('subject', []).append({
'name':
service,
'qcode':
service,
'scheme':
cp.DISTRIBUTION,
})
jimi = formatter.format(parsed, self.subscriber)[0][1]
root = etree.fromstring(jimi.encode(formatter.ENCODING))
return root.find('ContentItem')
def test_parse_agate_headings(self):
with open(get_fixture_path("ap-agate.json", "ap")) as fp:
_data = json.load(fp)
with self.app.app_context():
xml = etree.parse(get_fixture_path("ap-agate-nitf.xml", "ap"))
parsed = nitf.NITFFeedParser().parse(xml)
_data["nitf"] = parsed
with patch.dict(superdesk.resources, resources):
item = parser.parse(_data, {})
self.assertIn("<p>Atlantic Division</p>", item["body_html"])
def test_aps_category(self):
with open(get_fixture_path("ap-aps.json", "ap")) as fp:
_data = json.load(fp)
with self.app.app_context():
with patch.dict(superdesk.resources, resources):
item = parser.parse(_data, {})
self.assertEqual("Advisory", item["anpa_category"][0]["name"])
def test_category_tennis(self):
with open(get_fixture_path("ap-sports.json", "ap")) as fp:
_data = json.load(fp)
with self.app.app_context():
with patch.dict(superdesk.resources, resources):
item = parser.parse(_data, {})
self.assertEqual(
[{
"name": "Agate",
"qcode": "r",
"scheme": CATEGORY_SCHEME,
"translations": {
"name": {
"en-CA": "Agate",
"fr-CA": "Statistiques"
}
}
}],
item["anpa_category"],
)
self.assertEqual([], [
s["name"]
for s in item["subject"] if s.get("scheme") == AP_SUBJECT_CV
])
output = self.format(item)
self.assertIn("<Category>Agate</Category>", output)
self.assertIn("<IndexCode>Agate</IndexCode>", output)
def test_ignore_slugline_to_subject_map(self):
with open(get_fixture_path("ap-sports.json", "ap")) as fp:
_data = json.load(fp)
# Prefix slugline with `BC` so slugline -> subject mapping works
# in this case, slugline -> "BC-TEN-" -> "15065000"
_data["data"]["item"][
"slugline"] = "BC" + _data["data"]["item"]["slugline"][2:]
with self.app.app_context():
with patch.dict(superdesk.resources, resources):
item = parser.parse(_data, {})
self.assertEqual(
[{
"name": "Agate",
"qcode": "r",
"scheme": CATEGORY_SCHEME,
"translations": {
"name": {
"en-CA": "Agate",
"fr-CA": "Statistiques"
}
}
}],
item["anpa_category"],
)
self.assertEqual([], [
s["name"]
for s in item["subject"] if s.get("scheme") == AP_SUBJECT_CV
])
output = self.format(item)
self.assertIn("<Category>Agate</Category>", output)
# Make sure `IndexCode` only contains `Agate` and not `Sport` or `Tennis`
self.assertIn("<IndexCode>Agate</IndexCode>", output)
def test_parse_table(self):
with open(get_fixture_path("ap-table.json", "ap")) as fp:
_data = json.load(fp)
with self.app.app_context():
xml = etree.parse(get_fixture_path("ap-table-nitf.xml", "ap"))
parsed = nitf.NITFFeedParser().parse(xml)
_data["nitf"] = parsed
with patch.dict(superdesk.resources, resources):
item = parser.parse(_data, {})
self.assertIn("<table>", item["body_html"])
output = self.format(item)
jimi = etree.fromstring(output.encode("utf-8"))
print("jimi", jimi)
content = jimi.find("ContentItem").find("ContentText").text
self.assertIn("table", content)
def test_parse_aps_right_now(self):
with open(get_fixture_path("ap-aps-mi-right-now.json", "ap")) as fp:
_data = json.load(fp)
with self.app.app_context():
with patch.dict(superdesk.resources, resources):
item = parser.parse(_data, {})
self.assertEqual("International", item["anpa_category"][0]["name"])
def test_parse_subject_duplicates(self):
with open(get_fixture_path("ap-subject.json", "ap")) as fp:
_data = json.load(fp)
with self.app.app_context():
with patch.dict(superdesk.resources, resources):
item = parser.parse(_data, {})
qcodes = [subj["qcode"] for subj in item["subject"]]
self.assertEqual(len(qcodes), len(set(qcodes)))
def test_slugline_prev_version(self):
with open(get_fixture_path("ap-sports.json", "ap")) as fp:
_data = json.load(fp)
with self.app.app_context():
with patch.dict(superdesk.resources, resources):
resources["ingest"].service.find_one.return_value = {
"slugline": "prev-slugline",
}
item = parser.parse(_data, {})
resources["ingest"].service.find_one.return_value = None
self.assertEqual("prev-slugline", item["slugline"])
def test_category(self):
with open(get_fixture_path('politics.json', 'ap')) as fp:
_data = json.load(fp)
with self.app.app_context():
with patch.dict(superdesk.resources, resources):
item = parser.parse(_data, {})
self.assertIn({
'name': 'International',
'qcode': 'w',
'scheme': CATEGORY_SCHEME,
}, item['anpa_category'])
def test_category_tenis(self):
with open(get_fixture_path('ap-sports.json', 'ap')) as fp:
_data = json.load(fp)
with self.app.app_context():
with patch.dict(superdesk.resources, resources):
item = parser.parse(_data, {})
self.assertEqual([{
'name': 'Agate',
'qcode': 'r',
'scheme': CATEGORY_SCHEME,
}], item['anpa_category'])
def test_category_apv(self):
with open(get_fixture_path('apv.json', 'ap')) as fp:
_data = json.load(fp)
with self.app.app_context():
with patch.dict(superdesk.resources, resources):
item = parser.parse(_data, {})
self.assertEqual([{
'name': 'International',
'qcode': 'w',
'scheme': CATEGORY_SCHEME,
}], item['anpa_category'])
self.assertEqual('EU-Spain-Storm-Aftermath', item['slugline'])
def parse_format(self, source, binary=None, service=None):
with open(get_fixture_path(source, "ap")) as fp:
data = json.load(fp)
with self.app.app_context():
with patch.dict(superdesk.resources, resources):
with requests_mock.mock() as mock:
if binary:
with open(get_fixture_path(binary, "ap"), "rb") as f:
mock.get(
data["data"]["item"]["renditions"]["preview"]
["href"],
content=f.read(),
)
parsed = parser.parse(data, self.provider)
parsed["_id"] = "superdesk-id"
parsed["guid"] = "superdesk-guid"
parsed["unique_id"] = 1
parsed["family_id"] = parsed["_id"]
parsed["renditions"] = {
"original": {
"media": "abcd-media",
"mimetype": "image/jpeg"
}
}
if service:
parsed.setdefault("subject", []).append({
"name":
service,
"qcode":
service,
"scheme":
cp.DISTRIBUTION,
})
jimi = formatter.format(parsed, self.subscriber)[0][1]
root = etree.fromstring(jimi.encode(formatter.ENCODING))
return root.find("ContentItem")
def test_parse_picture(self):
with self.app.app_context():
with patch.dict(superdesk.resources, resources):
with requests_mock.mock() as mock:
with open(get_fixture_path('preview.jpg', 'ap'), 'rb') as f:
mock.get(picture_data['data']['item']['renditions']['preview']['href'], content=f.read())
item = parser.parse(picture_data, provider)
self.assertEqual('Jae C. Hong', item['byline'])
self.assertEqual(5, item['urgency'])
self.assertEqual('ASSOCIATED PRESS', item['creditline'])
self.assertEqual('America Protests Racial Economics', item['headline'])
self.assertEqual('stf', item['extra']['photographer_code'])
self.assertIn('Pedestrians are silhouetted', item['description_text'])
self.assertEqual('AP', item['extra']['provider'])
def test_category_apv(self):
with open(get_fixture_path("apv.json", "ap")) as fp:
_data = json.load(fp)
with self.app.app_context():
with patch.dict(superdesk.resources, resources):
item = parser.parse(_data, {})
self.assertEqual(
[{
"name": "International",
"qcode": "w",
"scheme": CATEGORY_SCHEME,
}],
item["anpa_category"],
)
self.assertEqual("EU-Spain-Storm-Aftermath", item["slugline"])
def test_parse_picture(self):
with self.app.app_context():
with patch.dict(superdesk.resources, resources):
with requests_mock.mock() as mock:
with open(get_fixture_path("preview.jpg", "ap"),
"rb") as f:
mock.get(
picture_data["data"]["item"]["renditions"]
["preview"]["href"],
content=f.read(),
)
item = parser.parse(picture_data, provider)
self.assertEqual("Jae C. Hong", item["byline"])
self.assertEqual(5, item["urgency"])
self.assertEqual("ASSOCIATED PRESS", item["creditline"])
self.assertEqual("America Protests Racial Economics", item["headline"])
self.assertEqual("stf", item["extra"]["photographer_code"])
self.assertIn("Pedestrians are silhouetted", item["description_text"])
self.assertEqual("AP", item["extra"]["provider"])
def test_category_politics_international(self):
with open(get_fixture_path("politics.json", "ap")) as fp:
_data = json.load(fp)
with self.app.app_context():
with patch.dict(superdesk.resources, resources):
item = parser.parse(_data, {})
self.assertEqual(
[{
"name": "International",
"qcode": "w",
"scheme": CATEGORY_SCHEME,
"translations": {
"name": {
"en-CA": "International",
"fr-CA": "International"
}
}
}],
item["anpa_category"],
)
self.assertEqual("US-Biden-Staff", item["slugline"])
import json
import flask
import unittest
import superdesk
import requests_mock
from datetime import datetime, timedelta
from unittest.mock import MagicMock, patch
from superdesk.metadata.item import SCHEDULE_SETTINGS, PUB_STATUS
from tests.ingest.parser import get_fixture_path
from tests.mock import resources
from cp.ingest import CP_APMediaFeedParser
with open(get_fixture_path('item.json', 'ap')) as fp:
data = json.load(fp)
with open(get_fixture_path('picture.json', 'ap')) as fp:
picture_data = json.load(fp)
provider = {}
parser = CP_APMediaFeedParser()
class CP_AP_ParseTestCase(unittest.TestCase):
app = flask.Flask(__name__)
app.locators = MagicMock()
def test_slugline(self):
import settings
import lxml.etree as etree
from datetime import datetime, timedelta
from unittest.mock import MagicMock, patch
from superdesk.metadata.item import SCHEDULE_SETTINGS, PUB_STATUS
from tests.ingest.parser import get_fixture_path
from superdesk.io.feed_parsers import nitf
from tests.mock import resources
from cp.ingest import CP_APMediaFeedParser
from cp.ingest.parser.ap import AP_SUBJECT_CV, CATEGORY_SCHEME
from cp.output.formatter.jimi import JimiFormatter
with open(get_fixture_path("item.json", "ap")) as fp:
data = json.load(fp)
with open(get_fixture_path("picture.json", "ap")) as fp:
picture_data = json.load(fp)
provider = {}
parser = CP_APMediaFeedParser()
class CP_AP_ParseTestCase(unittest.TestCase):
app = flask.Flask(__name__)
app.locators = MagicMock()
app.config.update({"AP_TAGS_MAPPING": settings.AP_TAGS_MAPPING})
subscriber = {}
