def setUp(self): dirname = os.path.dirname(os.path.realpath(__file__)) fixture = os.path.normpath(os.path.join(dirname, '../fixtures', self.filename)) provider = {'name': 'Wenn'} with open(fixture, 'rb') as f: self.file = f.read() etree.fromstring(self.file) self.items = WENNFeedParser().parse(etree.fromstring(self.file), provider)
def setUp(self): dirname = os.path.dirname(os.path.realpath(__file__)) fixture = os.path.join(dirname, 'fixtures', self.filename) provider = {'name': 'Wenn'} with open(fixture) as f: self.file = f.read() etree.fromstring(self.file) self.items = WENNParser().parse_message(etree.fromstring(self.file), provider)
def setUp(self): dirname = os.path.dirname(os.path.realpath(__file__)) fixture = os.path.join(dirname, 'fixtures', self.filename) provider = {'name': 'Test'} with open(fixture) as f: self.nitf = f.read() self.item = NITFParser().parse_message(etree.fromstring(self.nitf), provider)
def setUpFixture(self, filename): dirname = os.path.dirname(os.path.realpath(__file__)) fixture = os.path.join(dirname, 'fixtures', filename) with open(fixture) as f: self.tree = etree.fromstring(f.read().encode('utf-8')) parser = newsml_2_0.NewsMLTwoParser() self.item = parser.parse_message(self.tree)[0]
def _get_tree(self, endpoint, payload=None): """Get xml response for given API endpoint and payload. :param: endpoint :type endpoint: str :param: payload :type payload: str """ if payload is None: payload = {} payload['token'] = self._get_auth_token(self.provider, update=True) url = self._get_absolute_url(endpoint) if not self.session: self.session = requests.Session() retries = 0 while True: try: response = self.session.get(url, params=payload, timeout=(30, 15)) except requests.exceptions.Timeout as ex: if retries < 3: logger.warn( 'Reuters API timeout retrying, retries {}'.format( retries)) retries += 1 continue raise IngestApiError.apiTimeoutError(ex, self.provider) except requests.exceptions.TooManyRedirects as ex: # Tell the user their URL was bad and try a different one raise IngestApiError.apiRedirectError(ex, self.provider) except requests.exceptions.RequestException as ex: # catastrophic error. bail. raise IngestApiError.apiRequestError(ex, self.provider) except Exception as error: traceback.print_exc() raise IngestApiError.apiGeneralError(error, self.provider) if response.status_code == 404: raise LookupError( _('Not found {payload}').format(payload=payload)) break try: return etree.fromstring( response.content) # workaround for http mock lib except UnicodeEncodeError as error: traceback.print_exc() raise IngestApiError.apiUnicodeError(error, self.provider) except ParseError as error: traceback.print_exc() raise IngestApiError.apiParseError(error, self.provider) except Exception as error: traceback.print_exc() raise IngestApiError.apiGeneralError(error, self.provider)
def setUp(self): dirname = os.path.dirname(os.path.realpath(__file__)) fixture = os.path.join(dirname, 'fixtures', 'afp.xml') provider = {'name': 'Test'} with open(fixture, 'rb') as f: self.item = AFPNewsMLOneFeedParser().parse( etree.fromstring(f.read()), provider)
def _generate_auth_token(self, provider): """ Generates Authentication Token as per the configuration in Ingest Provider. :param provider: dict - Ingest provider details to which the current directory has been configured :type provider: dict :py:class: `superdesk.io.ingest_provider_model.IngestProviderResource` :return: token details if successfully authenticated :rtype: str :raises: IngestApiError.apiGeneralError() if auth_url is missing in the Ingest Provider configuration """ session = requests.Session() session.mount('https://', SSLAdapter()) auth_url = provider.get('config', {}).get('auth_url', None) if not auth_url: raise IngestApiError.apiGeneralError(provider=provider, exception=KeyError(''' Ingest Provider {} is missing Authentication URL. Please check the configuration. '''.format( provider['name']))) payload = { 'username': provider.get('config', {}).get('username', ''), 'password': provider.get('config', {}).get('password', ''), } response = session.get(auth_url, params=payload, verify=False, timeout=30) tree = etree.fromstring( response.content) # workaround for http mock lib return tree.text
def setUp(self): dirname = os.path.dirname(os.path.realpath(__file__)) fixture = os.path.normpath(os.path.join(dirname, '../fixtures', self.filename)) provider = {'name': 'Test'} with open(fixture) as f: self.scoop = f.read() self.item = ScoopNewsMLTwoFeedParser().parse(etree.fromstring(self.scoop), provider)
def _update(self, provider): self.provider = provider self.path = provider.get('config', {}).get('path', None) if not self.path: return [] for filename in get_sorted_files(self.path, sort_by=FileSortAttributes.created): try: filepath = os.path.join(self.path, filename) if os.path.isfile(filepath): stat = os.lstat(filepath) last_updated = datetime.fromtimestamp(stat.st_mtime, tz=utc) if self.is_latest_content(last_updated, provider.get('last_updated')): with open(filepath, 'r') as f: item = self.parser.parse_message(etree.fromstring(f.read()), provider) self.move_file(self.path, filename, provider=provider, success=True) yield [item] else: self.move_file(self.path, filename, provider=provider, success=True) except etreeParserError as ex: logger.exception("Ingest Type: AAP - File: {0} could not be processed".format(filename)) self.move_file(self.path, filename, provider=provider, success=False) raise ParserError.nitfParserError(ex, provider) except ParserError as ex: self.move_file(self.path, filename, provider=provider, success=False) except Exception as ex: self.move_file(self.path, filename, provider=provider, success=False) raise ProviderError.ingestError(ex, provider) push_notification('ingest:update')
def _generate_auth_token(self, provider): """ Generates Authentication Token as per the configuration in Ingest Provider. :param provider: dict - Ingest provider details to which the current directory has been configured :type provider: dict :py:class: `superdesk.io.ingest_provider_model.IngestProviderResource` :return: token details if successfully authenticated :rtype: str :raises: IngestApiError.apiGeneralError() if auth_url is missing in the Ingest Provider configuration """ session = requests.Session() session.mount('https://', SSLAdapter()) auth_url = provider.get('config', {}).get('auth_url', None) if not auth_url: raise IngestApiError.apiGeneralError(provider=provider, exception=KeyError( ''' Ingest Provider {} is missing Authentication URL. Please check the configuration. '''.format(provider['name'])) ) payload = { 'username': provider.get('config', {}).get('username', ''), 'password': provider.get('config', {}).get('password', ''), } response = session.get(auth_url, params=payload, verify=False, timeout=30) if response.status_code < 200 or response.status_code >= 300: raise IngestApiError.apiAuthError(provider=provider) tree = etree.fromstring(response.content) # workaround for http mock lib return tree.text
def _parse_file(self, filename): dirname = os.path.dirname(os.path.realpath(__file__)) fixture = os.path.normpath(os.path.join(dirname, '../fixtures', filename)) provider = {'name': 'Test'} with open(fixture, 'rb') as f: self.root_elt = etree.fromstring(f.read()) self.item = RitzauFeedParser().parse(self.root_elt, provider)
def setUp(self): dirname = os.path.dirname(os.path.realpath(__file__)) fixture = os.path.normpath( os.path.join(dirname, "../fixtures", "ana1.xml")) provider = {"name": "Test", "source": "ANA"} with open(fixture, "rb") as f: self.item = ANANewsMLOneFeedParser().parse( etree.fromstring(f.read()), provider)
def _parse_file(self, filename): dirname = os.path.dirname(os.path.realpath(__file__)) fixture = os.path.normpath( os.path.join(dirname, '../fixtures', filename)) provider = {'name': 'Test'} with open(fixture, 'rb') as f: self.root_elt = etree.fromstring(f.read()) self.item = RitzauFeedParser().parse(self.root_elt, provider)
def setUp(self): config.NITF_MAPPING = self.mapping dirname = os.path.dirname(os.path.realpath(__file__)) fixture = os.path.normpath(os.path.join(dirname, '../fixtures', self.filename)) provider = {'name': 'Test'} with open(fixture) as f: self.nitf = f.read() self.item = NITFFeedParser().parse(etree.fromstring(self.nitf), provider)
def setUp(self): super().setUp() config.NITF_MAPPING = self.mapping dirname = os.path.dirname(os.path.realpath(__file__)) fixture = os.path.normpath(os.path.join(dirname, '../fixtures', self.filename)) provider = {'name': 'Test'} with open(fixture) as f: self.nitf = f.read() self.item = NITFFeedParser().parse(etree.fromstring(self.nitf), provider)
def setUp(self): with self.app.app_context(): self.app.data.insert('vocabularies', self.vocab) dirname = os.path.dirname(os.path.realpath(__file__)) fixture = os.path.normpath(os.path.join(dirname, '../fixtures', self.filename)) provider = {'name': 'Test'} with open(fixture, 'rb') as f: self.root_elt = etree.fromstring(f.read()) self.item = RitzauFeedParser().parse(self.root_elt, provider)
def setUp(self): with self.app.app_context(): self.app.data.insert('vocabularies', self.vocab) dirname = os.path.dirname(os.path.realpath(__file__)) fixture = os.path.normpath(os.path.join(dirname, '../fixtures', self.filename)) provider = {'name': 'Test'} with open(fixture) as f: self.nitf = f.read() self.item = NITFFeedParser().parse(etree.fromstring(self.nitf), provider)
def _run_parse(self): dirname = os.path.dirname(os.path.realpath(__file__)) fixture = os.path.normpath( os.path.join(dirname, '../fixtures', self.filename)) provider = {'name': 'Test'} with open(fixture, 'rb') as f: self.xml = f.read() self.xml_root = etree.fromstring(self.xml) self.item = self.parser.parse(self.xml_root, provider)
def setUp(self): with self.app.app_context(): self.app.data.insert('vocabularies', self.vocab) dirname = os.path.dirname(os.path.realpath(__file__)) fixture = os.path.normpath(os.path.join(dirname, '../fixtures', self.filename)) provider = {'name': 'Test'} with open(fixture, 'rb') as f: self.root_elt = etree.fromstring(f.read()) self.items = NewsMLTwoFeedParser().parse(self.root_elt, provider)
def setUp(self): dirname = os.path.dirname(os.path.realpath(__file__)) fixture = os.path.normpath( os.path.join(dirname, "../fixtures", self.filename)) provider = {"name": "Test"} with open(fixture) as f: self.nitf = f.read() self.item = EFEFeedParser().parse(etree.fromstring(self.nitf), provider)
def setUp(self): dirname = os.path.dirname(os.path.realpath(__file__)) fixture = os.path.normpath( os.path.join(dirname, '../fixtures', self.filename)) provider = {'name': 'Test'} with open(fixture) as f: self.scoop = f.read() self.item = ScoopNewsMLTwoFeedParser().parse( etree.fromstring(self.scoop), provider)
def setUp(self): super().setUp() with self.app.app_context(): self.app.data.insert('vocabularies', self.vocab) dirname = os.path.dirname(os.path.realpath(__file__)) fixture = os.path.normpath(os.path.join(dirname, '../fixtures', self.filename)) provider = {'name': 'Test'} with open(fixture) as f: self.nitf = f.read() self.item = NITFFeedParser().parse(etree.fromstring(self.nitf), provider)
def setUp(self): config.NITF_MAPPING = self.mapping dirname = os.path.dirname(os.path.realpath(__file__)) fixture = os.path.normpath( os.path.join(dirname, "../fixtures", self.filename)) provider = {"name": "Test"} with open(fixture, "rb") as f: self.nitf = f.read() self.item = NITFFeedParser().parse(etree.fromstring(self.nitf), provider)
def test_get_subjects_with_invalid_qcode(self): xml = ('<?xml version="1.0" encoding="UTF-8"?>' '<nitf><head>' '<tobject tobject.type="News">' '<tobject.property tobject.property.type="Current" />' '<tobject.subject tobject.subject.refnum="00000000" ' 'tobject.subject.type="Justice" tobject.subject.matter="Police" />' '</tobject></head></nitf>') subjects = NITFFeedParser().get_subjects(etree.fromstring(xml)) self.assertEqual(len(subjects), 0)
def __init__(self, methodname): super().__init__(methodname) dirname = os.path.dirname(os.path.realpath(__file__)) fixture = os.path.normpath( os.path.join(dirname, '../fixtures', self.filename)) provider = {'name': 'Test'} with open(fixture, 'rb') as f: self.ori_file = f.read() self.articles = wordpress_wxr.WPWXRFeedParser().parse( etree.fromstring(self.ori_file), provider)
def test_fixture_list_with_no_dates(self): filename = 'aap_cricket.xml' dirname = os.path.dirname(os.path.realpath(__file__)) fixture = os.path.normpath(os.path.join(dirname, '../fixtures', filename)) with open(fixture, 'rb') as f: self.xml = f.read() fixture = {'fixture_xml': etree.fromstring(self.xml), 'sport_id': '3', 'sport_name': 'Cricket', 'comp_name': 'Champions Trophy in England/Wales', 'comp_id': 'dom-436'} items = AAPSportsFixturesParser().parse(fixture, None) self.assertTrue(len(items) == 2)
def test_fixtures(self): filename = 'aap_soccer.xml' dirname = os.path.dirname(os.path.realpath(__file__)) fixture = os.path.normpath(os.path.join(dirname, '../fixtures', filename)) with open(fixture, 'rb') as f: self.xml = f.read() fixture = {'fixture_xml': etree.fromstring(self.xml), 'sport_id': '4', 'sport_name': 'Soccer', 'comp_name': 'Qualifiers', 'comp_id': 'int-314'} items = AAPSportsFixturesParser().parse(fixture, None) self.assertTrue(len(items) == 5)
def setUp(self): with self.app.app_context(): self.app.data.insert("vocabularies", self.vocab) dirname = os.path.dirname(os.path.realpath(__file__)) fixture = os.path.normpath( os.path.join(dirname, "../fixtures", self.filename)) provider = {"name": "Test"} with open(fixture, "rb") as f: self.root_elt = etree.fromstring(f.read()) self.items = NewsMLTwoFeedParser().parse(self.root_elt, provider)
def test_get_subjects_with_invalid_qcode(self): xml = ('<?xml version="1.0" encoding="UTF-8"?>' '<nitf><head>' '<tobject tobject.type="News">' '<tobject.property tobject.property.type="Current" />' '<tobject.subject tobject.subject.refnum="00000000" ' 'tobject.subject.type="Justice" tobject.subject.matter="Police" />' '</tobject></head></nitf>') subjects = get_subjects(etree.fromstring(xml)) self.assertEqual(len(subjects), 0)
def __init__(self, methodname): super().__init__(methodname) dirname = os.path.dirname(os.path.realpath(__file__)) fixture = os.path.normpath(os.path.join(dirname, "../fixtures", self.filename)) provider = {"name": "Test"} with open(fixture, "rb") as f: buf = f.read() self.ori_file = buf parser = etree.XMLParser(recover=True) parsed = etree.fromstring(buf, parser) self.articles = wordpress_wxr.WPWXRFeedParser().parse(parsed, provider)
def _get_tree(self, endpoint, payload=None): """Get xml response for given API endpoint and payload. :param: endpoint :type endpoint: str :param: payload :type payload: str """ if payload is None: payload = {} payload['token'] = self._get_auth_token(self.provider, update=True) url = self._get_absolute_url(endpoint) if not self.session: self.session = requests.Session() retries = 0 while True: try: response = self.session.get(url, params=payload, timeout=(30, 15)) except requests.exceptions.Timeout as ex: if retries < 3: logger.warn('Reuters API timeout retrying, retries {}'.format(retries)) retries += 1 continue raise IngestApiError.apiTimeoutError(ex, self.provider) except requests.exceptions.TooManyRedirects as ex: # Tell the user their URL was bad and try a different one raise IngestApiError.apiRedirectError(ex, self.provider) except requests.exceptions.RequestException as ex: # catastrophic error. bail. raise IngestApiError.apiRequestError(ex, self.provider) except Exception as error: traceback.print_exc() raise IngestApiError.apiGeneralError(error, self.provider) if response.status_code == 404: raise LookupError('Not found %s' % payload) break try: return etree.fromstring(response.content) # workaround for http mock lib except UnicodeEncodeError as error: traceback.print_exc() raise IngestApiError.apiUnicodeError(error, self.provider) except ParseError as error: traceback.print_exc() raise IngestApiError.apiParseError(error, self.provider) except Exception as error: traceback.print_exc() raise IngestApiError.apiGeneralError(error, self.provider)
def test_fixture_list_with_dates(self): filename = 'aap_golf.xml' dirname = os.path.dirname(os.path.realpath(__file__)) fixture = os.path.normpath(os.path.join(dirname, '../fixtures', filename)) with open(fixture, 'rb') as f: self.xml = f.read() fixture = {'fixture_xml': etree.fromstring(self.xml), 'sport_id': '8', 'sport_name': 'Golf', 'comp_name': 'D+D Real Czech Masters', 'comp_id': 'int-26669'} items = AAPSportsFixturesParser().parse(fixture, None) self.assertTrue(len(items) == 1) self.assertEqual(items[0].get('calendars')[0].get('qcode'), 'sport')
def test_fixture_list_with_dates(self): filename = 'aap_golf.xml' dirname = os.path.dirname(os.path.realpath(__file__)) fixture = os.path.normpath(os.path.join(dirname, '../fixtures', filename)) with open(fixture, 'rb') as f: self.xml = f.read() fixture = {'fixture_xml': etree.fromstring(self.xml), 'sport_id': '8', 'sport_name': 'Golf', 'comp_name': 'D+D Real Czech Masters', 'comp_id': 'int-26669'} items = AAPSportsFixturesParser().parse(fixture, None) self.assertTrue(len(items) == 1) self.assertEqual(items[0].get('calendars')[0].get('qcode'), 'sport') self.assertEqual(items[0].get('calendars')[1].get('qcode'), 'sportgeneral')
def __init__(self, methodname): super().__init__(methodname) dirname = os.path.dirname(os.path.realpath(__file__)) fixture = os.path.normpath(os.path.join(dirname, '../fixtures', self.filename)) provider = {'name': 'Test'} with open(fixture, 'rb') as f: buf = f.read() self.ori_file = buf buf = buf.replace(b'\r', b' ') parser = etree.XMLParser(recover=True) parsed = etree.fromstring(buf, parser) self.articles = wordpress_wxr.WPWXRFeedParser().parse(parsed, provider)
def _update(self, provider): self.provider = provider self.path = provider.get('config', {}).get('path', None) if not self.path: return for filename in get_sorted_files(self.path, sort_by=FileSortAttributes.created): try: if os.path.isfile(os.path.join(self.path, filename)): filepath = os.path.join(self.path, filename) stat = os.lstat(filepath) last_updated = datetime.fromtimestamp(stat.st_mtime, tz=utc) if self.is_latest_content(last_updated, provider.get('last_updated')): with open(os.path.join(self.path, filename), 'r') as f: item = self.parser.parse_message( etree.fromstring(f.read()), provider) self.add_timestamps(item) self.move_file(self.path, filename, provider=provider, success=True) yield [item] else: self.move_file(self.path, filename, provider=provider, success=True) except etreeParserError as ex: logger.exception( "Ingest Type: AFP - File: {0} could not be processed". format(filename), ex) self.move_file(self.path, filename, provider=provider, success=False) raise ParserError.newsmlOneParserError(ex, provider) except ParserError as ex: self.move_file(self.path, filename, provider=provider, success=False) except Exception as ex: self.move_file(self.path, filename, provider=provider, success=False) raise ProviderError.ingestError(ex, provider) push_notification('ingest:update')
def test_get_subjects(self): xml = ('<?xml version="1.0" encoding="UTF-8"?>' '<nitf><head>' '<tobject tobject.type="News">' '<tobject.property tobject.property.type="Current" />' '<tobject.subject tobject.subject.refnum="02003000" ' 'tobject.subject.type="Justice" tobject.subject.matter="Police" />' '</tobject></head></nitf>') subjects = get_subjects(etree.fromstring(xml)) self.assertEqual(len(subjects), 2) self.assertIn({'qcode': '02000000', 'name': 'Justice'}, subjects) self.assertIn({'qcode': '02003000', 'name': 'Police'}, subjects)
def test_get_subjects(self): xml = ('<?xml version="1.0" encoding="UTF-8"?>' '<nitf><head>' '<tobject tobject.type="News">' '<tobject.property tobject.property.type="Current" />' '<tobject.subject tobject.subject.refnum="02003000" ' 'tobject.subject.type="Justice" tobject.subject.matter="Police" />' '</tobject></head></nitf>') subjects = NITFFeedParser().get_subjects(etree.fromstring(xml)) self.assertEqual(len(subjects), 2) self.assertIn({'qcode': '02000000', 'name': 'Justice'}, subjects) self.assertIn({'qcode': '02003000', 'name': 'Police'}, subjects)
def __init__(self, methodname): super().__init__(methodname) dirname = os.path.dirname(os.path.realpath(__file__)) fixture = os.path.normpath( os.path.join(dirname, '../fixtures', self.filename)) provider = {'name': 'Test'} with open(fixture, 'rb') as f: buf = f.read() self.ori_file = buf buf = buf.replace(b'\r', b' ') parser = etree.XMLParser(recover=True) parsed = etree.fromstring(buf, parser) self.articles = wordpress_wxr.WPWXRFeedParser().parse(parsed, provider)
def test_get_subjects(self): xml = ( b'<?xml version="1.0" encoding="UTF-8"?>' b"<nitf><head>" b'<tobject tobject.type="News">' b'<tobject.property tobject.property.type="Current" />' b'<tobject.subject tobject.subject.refnum="02003000" ' b'tobject.subject.type="Justice" tobject.subject.matter="Police" />' b"</tobject></head></nitf>") subjects = NITFFeedParser().get_subjects(etree.fromstring(xml)) self.assertEqual(len(subjects), 2) self.assertIn({"qcode": "02000000", "name": "Justice"}, subjects) self.assertIn({"qcode": "02003000", "name": "Police"}, subjects)
def test_get_subjects(self): xml = ( '<?xml version="1.0" encoding="UTF-8"?>' "<nitf><head>" '<tobject tobject.type="News">' '<tobject.property tobject.property.type="Current" />' '<tobject.subject tobject.subject.refnum="02003000" ' 'tobject.subject.type="Justice" tobject.subject.matter="Police" />' "</tobject></head></nitf>" ) subjects = get_subjects(etree.fromstring(xml)) self.assertEqual(len(subjects), 2) self.assertIn({"qcode": "02000000", "name": "Justice"}, subjects) self.assertIn({"qcode": "02003000", "name": "Police"}, subjects)
def parse_file(self, filename, provider): try: self.path = provider.get('config', {}).get('path', None) if not self.path: return [] with open(os.path.join(self.path, filename), 'r') as f: item = self.parser.parse_message(etree.fromstring(f.read()), provider) return [item] except Exception as ex: self.move_file(self.path, filename, provider=provider, success=False) raise ParserError.parseFileError('AAP', filename, ex, provider)
def fetch_token_from_api(provider): session = requests.Session() session.mount('https://', SSLAdapter()) url = 'https://commerce.reuters.com/rmd/rest/xml/login' payload = { 'username': provider.get('config', {}).get('username', ''), 'password': provider.get('config', {}).get('password', ''), } response = session.get(url, params=payload, stream=False, verify=False) # workaround for httmock lib # tree = etree.fromstring(response.text) tree = etree.fromstring(response.content) return tree.text
def fetch_token_from_api(provider): session = requests.Session() session.mount("https://", SSLAdapter()) url = "https://commerce.reuters.com/rmd/rest/xml/login" payload = { "username": provider.get("config", {}).get("username", ""), "password": provider.get("config", {}).get("password", ""), } response = session.get(url, params=payload, stream=False, verify=False) # workaround for httmock lib # tree = etree.fromstring(response.text) tree = etree.fromstring(response.content) return tree.text
def fetch_token_from_api(provider): session = requests.Session() session.mount('https://', SSLAdapter()) url = 'https://commerce.reuters.com/rmd/rest/xml/login' payload = { 'username': provider.get('config', {}).get('username', ''), 'password': provider.get('config', {}).get('password', ''), } response = session.get(url, params=payload, verify=False, timeout=30) # workaround for httmock lib # tree = etree.fromstring(response.text) tree = etree.fromstring(response.content) return tree.text
def setUp(self): super().setUp() # we need to prepopulate vocabularies to get qcodes voc_file = os.path.join(os.path.abspath(os.path.dirname(settings.__file__)), "data/vocabularies.json") VocabulariesPopulateCommand().run(voc_file) # settings are needed in order to get into account NITF_MAPPING for key in dir(settings): if key.isupper(): setattr(config, key, getattr(settings, key)) dirname = os.path.dirname(os.path.realpath(__file__)) fixture = os.path.normpath(os.path.join(dirname, '../fixtures', self.filename)) provider = {'name': 'Test'} with open(fixture) as f: self.nitf = f.read() self.item = NITFFeedParser().parse(etree.fromstring(self.nitf), provider)
def _get_tree(self, endpoint, payload=None): """ Get xml response for given API endpoint and payload. :param: endpoint :type endpoint: str :param: payload :type payload: str """ if payload is None: payload = {} payload['token'] = self._get_auth_token(self.provider, update=True) url = self._get_absolute_url(endpoint) try: response = requests.get(url, params=payload, timeout=15) except requests.exceptions.Timeout as ex: # Maybe set up for a retry, or continue in a retry loop raise IngestApiError.apiTimeoutError(ex, self.provider) except requests.exceptions.TooManyRedirects as ex: # Tell the user their URL was bad and try a different one raise IngestApiError.apiRedirectError(ex, self.provider) except requests.exceptions.RequestException as ex: # catastrophic error. bail. raise IngestApiError.apiRequestError(ex, self.provider) except Exception as error: traceback.print_exc() raise IngestApiError.apiGeneralError(error, self.provider) if response.status_code == 404: raise LookupError('Not found %s' % payload) try: return etree.fromstring( response.content) # workaround for http mock lib except UnicodeEncodeError as error: traceback.print_exc() raise IngestApiError.apiUnicodeError(error, self.provider) except ParseError as error: traceback.print_exc() raise IngestApiError.apiParseError(error, self.provider) except Exception as error: traceback.print_exc() raise IngestApiError.apiGeneralError(error, self.provider)
def _get_tree(self, endpoint, payload=None): """ Get xml response for given API endpoint and payload. :param: endpoint :type endpoint: str :param: payload :type payload: str """ if payload is None: payload = {} payload['token'] = self._get_auth_token(self.provider, update=True) url = self._get_absolute_url(endpoint) try: response = requests.get(url, params=payload, timeout=15) except requests.exceptions.Timeout as ex: # Maybe set up for a retry, or continue in a retry loop raise IngestApiError.apiTimeoutError(ex, self.provider) except requests.exceptions.TooManyRedirects as ex: # Tell the user their URL was bad and try a different one raise IngestApiError.apiRedirectError(ex, self.provider) except requests.exceptions.RequestException as ex: # catastrophic error. bail. raise IngestApiError.apiRequestError(ex, self.provider) except Exception as error: traceback.print_exc() raise IngestApiError.apiGeneralError(error, self.provider) if response.status_code == 404: raise LookupError('Not found %s' % payload) try: return etree.fromstring(response.content) # workaround for http mock lib except UnicodeEncodeError as error: traceback.print_exc() raise IngestApiError.apiUnicodeError(error, self.provider) except ParseError as error: traceback.print_exc() raise IngestApiError.apiParseError(error, self.provider) except Exception as error: traceback.print_exc() raise IngestApiError.apiGeneralError(error, self.provider)
def get_tree(self, endpoint, payload=None): """Get xml response for given API endpoint and payload.""" if payload is None: payload = {} payload['token'] = self.get_token() url = self.get_url(endpoint) try: response = requests.get(url, params=payload, timeout=21.0) except requests.exceptions.Timeout as ex: # Maybe set up for a retry, or continue in a retry loop raise IngestApiError.apiTimeoutError(ex, self.provider) except requests.exceptions.TooManyRedirects as ex: # Tell the user their URL was bad and try a different one raise IngestApiError.apiRedirectError(ex, self.provider) except requests.exceptions.RequestException as ex: # catastrophic error. bail. raise IngestApiError.apiRequestError(ex, self.provider) except Exception as error: traceback.print_exc() raise IngestApiError(error, self.provider) if response.status_code == 404: raise IngestApiError.apiNotFoundError(LookupError('Not found %s' % payload), self.provider) try: # workaround for httmock lib # return etree.fromstring(response.text.encode('utf-8')) return etree.fromstring(response.content) except UnicodeEncodeError as error: traceback.print_exc() raise IngestApiError.apiUnicodeError(error, self.provider) except ParseError as error: traceback.print_exc() raise IngestApiError.apiParseError(error, self.provider) except Exception as error: traceback.print_exc() raise IngestApiError(error, self.provider)
def get_tree(self, endpoint, payload=None): """Get xml response for given API endpoint and payload.""" if payload is None: payload = {} payload['token'] = self.get_token() url = self.get_url(endpoint) try: response = requests.get(url, params=payload, timeout=21.0) except Exception as error: traceback.print_exc() raise error if response.status_code == 404: raise LookupError('Not found %s' % payload) try: # workaround for httmock lib # return etree.fromstring(response.text.encode('utf-8')) return etree.fromstring(response.content) except UnicodeEncodeError as error: traceback.print_exc() raise error
def setUp(self): dirname = os.path.dirname(os.path.realpath(__file__)) fixture = os.path.join(dirname, 'fixtures', 'afp.xml') provider = {'name': 'Test'} with open(fixture) as f: self.item = NewsMLOneParser().parse_message(etree.fromstring(f.read()), provider)
def setUp(self): dirname = os.path.dirname(os.path.realpath(__file__)) fixture = os.path.join(dirname, "fixtures", "afp.xml") provider = {"name": "Test"} with open(fixture) as f: self.item = NewsMLOneParser().parse_message(etree.fromstring(f.read()), provider)
def test_can_parse(self): self.assertTrue(ScoopNewsMLTwoFeedParser().can_parse(etree.fromstring(self.scoop)))
def get_etree(filename): dirname = os.path.dirname(os.path.realpath(__file__)) with open(os.path.join(dirname, 'fixtures', filename)) as f: return etree.fromstring(f.read().encode('utf-8'))
def setUp(self): dirname = os.path.dirname(os.path.realpath(__file__)) fixture = os.path.join(dirname, 'fixtures', 'afp.xml') with open(fixture) as f: self.item = Parser().parse_message(etree.fromstring(f.read()))