def test07_process_source(self):
# connection and readable resourcesync, write sitemap to file
try:
shutil.rmtree("rs/destination/d6/sitemaps")
except:
pass
Config.__set_config_filename__("test-files/config.txt")
Config().__drop__()
DestinationMap.__set_map_filename__("test-files/desmap.txt")
DestinationMap().__drop__()
des.reporter.reset_instance()
Config().__set_prop__(Config.key_use_netloc, "False")
Config().__set_prop__(Config.key_audit_only, "False")
DestinationMap().__set_destination__(
"http://localhost:8000/rs/source/s6", "rs/destination/d6")
des.processor.processor_listeners.append(SitemapWriter())
base_uri = "http://localhost:8000/rs/source/s6/"
sdproc = Sodesproc(base_uri)
sdproc.read_source()
self.assertEqual(200, sdproc.source_status)
self.assertEqual(Status.document, sdproc.status)
self.assertTrue(
os.path.isfile(
"rs/destination/d6/sitemaps/.well-known/resourcesync"))
def test05_process_source(self):
# connection and readable resourcesync but capability is not 'description'
base_uri = "http://localhost:8000/rs/source/s5/"
sdproc = Sodesproc(base_uri)
sdproc.read_source()
self.assertEqual(200, sdproc.source_status)
self.assertEqual(1, len(sdproc.exceptions))
self.assertEqual(Status.read_error, sdproc.status)
sdproc.process_source()
self.assertEqual(Status.read_error, sdproc.status)
def test04_process_source(self):
# connection and resourcesync is xml but not sitemap
base_uri = "http://localhost:8000/rs/source/s4/"
sdproc = Sodesproc(base_uri)
sdproc.read_source()
self.assertEqual(200, sdproc.source_status)
self.assertEqual(1, len(sdproc.exceptions))
self.assertEqual(Status.read_error, sdproc.status)
sdproc.process_source()
self.assertEqual(Status.read_error, sdproc.status)
def test02_process_source(self):
# connection but no .well-known/...
base_uri = "http://localhost:8000/rs/source/s2/"
sdproc = Sodesproc(base_uri)
sdproc.read_source()
self.assertEqual(404, sdproc.source_status)
self.assertEqual(1, len(sdproc.exceptions))
self.assertEqual(Status.read_error, sdproc.status)
sdproc.process_source()
self.assertEqual(Status.read_error, sdproc.status)
def try_wellknown(self):
"""
The uri can be extended with '.well-known/resourcesync' which leads to a valid source description.
:return: Sodesproc on a source description or None
"""
processor = Sodesproc(self.uri, report_errors=False)
processor.read_source()
if processor.status == Status.document:
processor.report_errors = True
return processor
else:
return None
def test03_process_source_(self):
# connection and unreadable resourcesync
base_uri = "http://localhost:8000/rs/source/s3/"
sdproc = Sodesproc(base_uri)
sdproc.read_source()
self.assertEqual(200, sdproc.source_status)
self.assertEqual(1, len(sdproc.exceptions))
self.assertEqual(Status.read_error, sdproc.status)
sdproc.process_source()
self.assertEqual(Status.read_error, sdproc.status)
def test01_process_source(self):
# no connection to non-existent uri
base_uri = "http://ditbestaatechtniet.com/out/there"
sdproc = Sodesproc(base_uri)
self.assertTrue(sdproc.base_uri.endswith("/"))
sdproc.read_source()
self.assertFalse(sdproc.source_status)
self.assertEqual(1, len(sdproc.exceptions))
self.assertEqual(Status.read_error, sdproc.status)
sdproc.process_source()
self.assertEqual(Status.read_error, sdproc.status)
def test06_process_source(self):
# connection and readable resourcesync
base_uri = "http://localhost:8000/rs/source/s6/"
sdproc = Sodesproc(base_uri)
sdproc.read_source()
self.assertEqual(200, sdproc.source_status)
self.assertEqual(Status.document, sdproc.status)
# sdproc.source_description is a resync.resource_container.ResourceContainer
self.assertEqual("http://example.com/info_about_source.xml", sdproc.describedby_url)
self.assertEqual(0, len(sdproc.exceptions))
self.assertEqual(Status.document, sdproc.status)
sdproc.process_source()
self.assertEqual(Status.processed_with_exceptions, sdproc.status)
self.assertEqual(3, len(sdproc.exceptions))
def __do_task__(self, task):
for uri in self.sources:
processor = None
if task == "discover":
discoverer = Discoverer(uri)
processor = discoverer.get_processor()
elif task == "wellknown":
processor = Sodesproc(uri)
elif task == "capability":
processor = Capaproc(uri)
if processor is None:
msg = "Could not discover processor for '%s'" % uri
self.logger.warn(msg)
self.exceptions.append(msg)
des.reporter.instance().log_status(uri, exception=msg)
else:
try:
processor.process_source()
self.exceptions.extend(processor.exceptions)
# do something with processor status
except Exception as err:
self.exceptions.append(err)
self.logger.warn("Failure while syncing %s" % uri, exc_info=True)
des.reporter.instance().log_status(uri, exception=err)
def test07_process_source(self):
# connection and readable resourcesync, write sitemap to file
try:
shutil.rmtree("rs/destination/d6/sitemaps")
except:
pass
Config.__set_config_filename__("test-files/config.txt")
Config().__drop__()
DestinationMap.__set_map_filename__("test-files/desmap.txt")
DestinationMap().__drop__()
des.reporter.reset_instance()
Config().__set_prop__(Config.key_use_netloc, "False")
Config().__set_prop__(Config.key_audit_only, "False")
DestinationMap().__set_destination__("http://localhost:8000/rs/source/s6", "rs/destination/d6")
des.processor.processor_listeners.append(SitemapWriter())
base_uri = "http://localhost:8000/rs/source/s6/"
sdproc = Sodesproc(base_uri)
sdproc.read_source()
self.assertEqual(200, sdproc.source_status)
self.assertEqual(Status.document, sdproc.status)
self.assertTrue(os.path.isfile("rs/destination/d6/sitemaps/.well-known/resourcesync"))
def test05_process_source(self):
# connection and readable resourcesync but capability is not 'description'
base_uri = "http://localhost:8000/rs/source/s5/"
sdproc = Sodesproc(base_uri)
sdproc.read_source()
self.assertEqual(200, sdproc.source_status)
self.assertEqual(1, len(sdproc.exceptions))
self.assertEqual(Status.read_error, sdproc.status)
sdproc.process_source()
self.assertEqual(Status.read_error, sdproc.status)
def test04_process_source(self):
# connection and resourcesync is xml but not sitemap
base_uri = "http://localhost:8000/rs/source/s4/"
sdproc = Sodesproc(base_uri)
sdproc.read_source()
self.assertEqual(200, sdproc.source_status)
self.assertEqual(1, len(sdproc.exceptions))
self.assertEqual(Status.read_error, sdproc.status)
sdproc.process_source()
self.assertEqual(Status.read_error, sdproc.status)
def test02_process_source(self):
# connection but no .well-known/...
base_uri = "http://localhost:8000/rs/source/s2/"
sdproc = Sodesproc(base_uri)
sdproc.read_source()
self.assertEqual(404, sdproc.source_status)
self.assertEqual(1, len(sdproc.exceptions))
self.assertEqual(Status.read_error, sdproc.status)
sdproc.process_source()
self.assertEqual(Status.read_error, sdproc.status)
def test03_process_source_(self):
# connection and unreadable resourcesync
base_uri = "http://localhost:8000/rs/source/s3/"
sdproc = Sodesproc(base_uri)
sdproc.read_source()
self.assertEqual(200, sdproc.source_status)
self.assertEqual(1, len(sdproc.exceptions))
self.assertEqual(Status.read_error, sdproc.status)
sdproc.process_source()
self.assertEqual(Status.read_error, sdproc.status)
def test01_process_source(self):
# no connection to non-existent uri
base_uri = "http://ditbestaatechtniet.com/out/there"
sdproc = Sodesproc(base_uri)
self.assertTrue(sdproc.base_uri.endswith("/"))
sdproc.read_source()
self.assertFalse(sdproc.source_status)
self.assertEqual(1, len(sdproc.exceptions))
self.assertEqual(Status.read_error, sdproc.status)
sdproc.process_source()
self.assertEqual(Status.read_error, sdproc.status)
def test06_process_source(self):
# connection and readable resourcesync
base_uri = "http://localhost:8000/rs/source/s6/"
sdproc = Sodesproc(base_uri)
sdproc.read_source()
self.assertEqual(200, sdproc.source_status)
self.assertEqual(Status.document, sdproc.status)
# sdproc.source_description is a resync.resource_container.ResourceContainer
self.assertEqual("http://example.com/info_about_source.xml",
sdproc.describedby_url)
self.assertEqual(0, len(sdproc.exceptions))
self.assertEqual(Status.document, sdproc.status)
sdproc.process_source()
self.assertEqual(Status.processed_with_exceptions, sdproc.status)
self.assertEqual(3, len(sdproc.exceptions))