def test07_try_robots(self):
uri = "http://localhost:8000/rs/source/discover/loc2"
discoverer = Discoverer(uri)
processor = discoverer.get_processor()
self.assertIsInstance(processor, proc.Reliproc)
processor.read_source()
def test02_capabilitylist(self):
uri = "http://localhost:8000/rs/source/discover/loc1/capabilitylist.xml"
discoverer = Discoverer(uri)
processor = discoverer.get_processor()
self.assertIsInstance(processor, proc.Capaproc)
self.assertEqual(processor.status, Status.document)
def __do_task__(self, task):
for uri in self.sources:
processor = None
if task == "discover":
discoverer = Discoverer(uri)
processor = discoverer.get_processor()
elif task == "wellknown":
processor = Sodesproc(uri)
elif task == "capability":
processor = Capaproc(uri)
if processor is None:
msg = "Could not discover processor for '%s'" % uri
self.logger.warn(msg)
self.exceptions.append(msg)
des.reporter.instance().log_status(uri, exception=msg)
else:
try:
processor.process_source()
self.exceptions.extend(processor.exceptions)
# do something with processor status
except Exception as err:
self.exceptions.append(err)
self.logger.warn("Failure while syncing %s" % uri, exc_info=True)
des.reporter.instance().log_status(uri, exception=err)
def test01_well_known(self):
uri = "http://localhost:8000/rs/source/discover/loc1"
discoverer = Discoverer(uri)
processor = discoverer.get_processor()
self.assertIsInstance(processor, proc.Sodesproc)
self.assertEqual(processor.status, Status.document)
def test07_try_robots(self):
uri = "http://localhost:8000/rs/source/discover/loc2"
discoverer = Discoverer(uri)
processor = discoverer.get_processor()
self.assertIsInstance(processor, proc.Reliproc)
processor.read_source()
def test02_capabilitylist(self):
uri = "http://localhost:8000/rs/source/discover/loc1/capabilitylist.xml"
discoverer = Discoverer(uri)
processor = discoverer.get_processor()
self.assertIsInstance(processor, proc.Capaproc)
self.assertEqual(processor.status, Status.document)
def test01_well_known(self):
uri = "http://localhost:8000/rs/source/discover/loc1"
discoverer = Discoverer(uri)
processor = discoverer.get_processor()
self.assertIsInstance(processor, proc.Sodesproc)
self.assertEqual(processor.status, Status.document)
def test08_try_robots_with_netloc(self):
DestinationMap().__remove_destination__("http://localhost:8000/rs/source/discover/")
Config().__set_prop__(Config.key_use_netloc, "True")
uri = "http://localhost:8000/rs/source/discover/loc2"
discoverer = Discoverer(uri)
processor = discoverer.get_processor()
self.assertIsInstance(processor, proc.Reliproc)
processor.read_source()
def test08_try_robots_with_netloc(self):
DestinationMap().__remove_destination__(
"http://localhost:8000/rs/source/discover/")
Config().__set_prop__(Config.key_use_netloc, "True")
uri = "http://localhost:8000/rs/source/discover/loc2"
discoverer = Discoverer(uri)
processor = discoverer.get_processor()
self.assertIsInstance(processor, proc.Reliproc)
processor.read_source()
def test06_try_link_unparsable_html(self):
uri = "http://localhost:8000/rs/source/discover/loc1/unparsable.html"
discoverer = Discoverer(uri)
processor = discoverer.get_processor()
self.assertIsNone(processor)
def test05_try_link_absent_html(self):
uri = "http://localhost:8000/rs/source/discover/loc1/no_page.html"
discoverer = Discoverer(uri)
processor = discoverer.get_processor()
self.assertIsNone(processor)
def test03_try_link_html(self):
uri = "http://localhost:8000/rs/source/discover/loc1/page.html"
discoverer = Discoverer(uri)
processor = discoverer.get_processor()
self.assertIsInstance(processor, proc.Capaproc)
def test06_try_link_unparsable_html(self):
uri = "http://localhost:8000/rs/source/discover/loc1/unparsable.html"
discoverer = Discoverer(uri)
processor = discoverer.get_processor()
self.assertIsNone(processor)
def test05_try_link_absent_html(self):
uri = "http://localhost:8000/rs/source/discover/loc1/no_page.html"
discoverer = Discoverer(uri)
processor = discoverer.get_processor()
self.assertIsNone(processor)
def test03_try_link_html(self):
uri = "http://localhost:8000/rs/source/discover/loc1/page.html"
discoverer = Discoverer(uri)
processor = discoverer.get_processor()
self.assertIsInstance(processor, proc.Capaproc)