def test07_process_source(self): # connection and readable resourcesync, write sitemap to file try: shutil.rmtree("rs/destination/d6/sitemaps") except: pass Config.__set_config_filename__("test-files/config.txt") Config().__drop__() DestinationMap.__set_map_filename__("test-files/desmap.txt") DestinationMap().__drop__() des.reporter.reset_instance() Config().__set_prop__(Config.key_use_netloc, "False") Config().__set_prop__(Config.key_audit_only, "False") DestinationMap().__set_destination__( "http://localhost:8000/rs/source/s6", "rs/destination/d6") des.processor.processor_listeners.append(SitemapWriter()) base_uri = "http://localhost:8000/rs/source/s6/" sdproc = Sodesproc(base_uri) sdproc.read_source() self.assertEqual(200, sdproc.source_status) self.assertEqual(Status.document, sdproc.status) self.assertTrue( os.path.isfile( "rs/destination/d6/sitemaps/.well-known/resourcesync"))
def test_set_filename_once(self): DestinationMap.__set_map_filename__("test-files/desmap.txt") self.assertEqual("test-files/desmap.txt", DestinationMap._get_map_filename()) desmap = DestinationMap() self.assertEqual("test-files/desmap.txt", desmap._map_filename) DestinationMap.__set_map_filename__("test-files/other-desmap.txt") self.assertEqual("test-files/desmap.txt", DestinationMap._get_map_filename()) desmap = DestinationMap() self.assertEqual("test-files/desmap.txt", desmap._map_filename)
def setUpModule(): global server server_address = ('', 8000) handler_class = SimpleHTTPRequestHandler server = HTTPServer(server_address, handler_class) t = threading.Thread(target=server.serve_forever) t.daemon = True logger.debug("Starting server at http://localhost:8000/") t.start() proc.processor_listeners.append(SitemapWriter()) Config.__set_config_filename__("test-files/config.txt") Config().__drop__() DestinationMap.__set_map_filename__("test-files/desmap.txt") DestinationMap().__drop__() DestinationMap().__set_destination__("http://localhost:8000/rs/source/discover/", "rs/destination/discover")
def setUpModule(): global server server_address = ('', 8000) handler_class = SimpleHTTPRequestHandler server = HTTPServer(server_address, handler_class) t = threading.Thread(target=server.serve_forever) t.daemon = True logger.debug("Starting server at http://localhost:8000/") t.start() proc.processor_listeners.append(SitemapWriter()) Config.__set_config_filename__("test-files/config.txt") Config().__drop__() DestinationMap.__set_map_filename__("test-files/desmap.txt") DestinationMap().__drop__() DestinationMap().__set_destination__( "http://localhost:8000/rs/source/discover/", "rs/destination/discover")
def run(self, sources, task="discover", once=False): """ Run the DesRunner. A running application can be stopped by creating a file named 'stop' in the directory the runner was started from. Source urls are read from the file given in param 'sources'. This file is read each time a full round of synchronizing has taken place, so source urls can be extended or changed without restarting the application. Sources are mapped to the destinations given in the file denoted by the configuration parameter "location_mapper_destination_file". :param sources: the file containing source urls :param task: the task to run. - If source urls can all be discovered by reading the .well-known/resourcesync on each source, use 'wellknown'. - If all source urls point to capability lists, use 'capability'. - If source urls are heterogeneous, use 'discover'. :param once: True for exploring source urls once and than exit, False otherwise :return: """ condition = True while condition: # list of urls self.logger.info("Reading source urls from '%s'" % sources) self.__read_sources_doc__(sources) # reset url --> destination map. New mappings may be configured DestinationMap.__set_map_filename__(Config(). prop(Config.key_location_mapper_destination_file, "conf/desmap.txt")) # drop to force fresh read from file DestinationMap().__drop__() # Set the root of the destination folder if configured DestinationMap().set_root_folder(Config().prop(Config.key_destination_root)) # do all the urls self.__do_task__(task) # report self.__do_report__(task) # to continue or not to continue condition = not (once or self.__stop__()) if condition: pause = Config().int_prop(Config.key_sync_pause) self.logger.info("Going to sleep for %d seconds." % pause) self.logger.debug("zzzzzzzzzzzzzzzzzzzzzzzzzzzzzz") time.sleep(pause) # repeat after sleep condition = not (once or self.__stop__())
def test07_process_source(self): # connection and readable resourcesync, write sitemap to file try: shutil.rmtree("rs/destination/d6/sitemaps") except: pass Config.__set_config_filename__("test-files/config.txt") Config().__drop__() DestinationMap.__set_map_filename__("test-files/desmap.txt") DestinationMap().__drop__() des.reporter.reset_instance() Config().__set_prop__(Config.key_use_netloc, "False") Config().__set_prop__(Config.key_audit_only, "False") DestinationMap().__set_destination__("http://localhost:8000/rs/source/s6", "rs/destination/d6") des.processor.processor_listeners.append(SitemapWriter()) base_uri = "http://localhost:8000/rs/source/s6/" sdproc = Sodesproc(base_uri) sdproc.read_source() self.assertEqual(200, sdproc.source_status) self.assertEqual(Status.document, sdproc.status) self.assertTrue(os.path.isfile("rs/destination/d6/sitemaps/.well-known/resourcesync"))
def test_find_destination(self): DestinationMap.__set_map_filename__("test-files/desmap.txt") desmap = DestinationMap() uri = "http://long.name.com/path/to/resource.xml" base_uri, destination = desmap.find_destination(uri) self.assertEqual("http://long.name.com", base_uri) self.assertEqual("./destination1", destination) uri = "http://long.name.com/path/to/" base_uri, destination = desmap.find_destination(uri) self.assertEqual("http://long.name.com", base_uri) self.assertEqual("./destination1", destination) uri = "http://long.name.com/" base_uri, destination = desmap.find_destination(uri) self.assertEqual("http://long.name.com", base_uri) self.assertEqual("./destination1", destination) uri = "http://long.name.com" base_uri, destination = desmap.find_destination(uri) self.assertEqual("http://long.name.com", base_uri) self.assertEqual("./destination1", destination) # explicit path to resource in desmap uri = "https://first.com:8080/path1" base_uri, destination = desmap.find_destination(uri) self.assertEqual("https://first.com:8080", base_uri) self.assertIsNone(destination) uri = "https://first.com:8080/path1/to/resource.xml" base_uri, destination = desmap.find_destination(uri) self.assertEqual("https://first.com:8080/path1/to/resource.xml", base_uri) self.assertEqual("./destination2", destination) uri = "https://first.com:8080/path2/" base_uri, destination = desmap.find_destination(uri) self.assertEqual("https://first.com:8080/path2", base_uri) self.assertEqual("./destination3", destination) uri = "https://first.com:8080/path2" base_uri, destination = desmap.find_destination(uri) self.assertEqual("https://first.com:8080/path2", base_uri) self.assertEqual("./destination3", destination) # uri = "https://not.mapped.com/resource.xml" base_uri, destination = desmap.find_destination(uri, "default/path") self.assertEqual("https://not.mapped.com", base_uri) self.assertEqual("./default/path", destination) # uri = "https://not.mapped.com/resource.xml" base_uri, destination = desmap.find_destination(uri, netloc=True) self.assertEqual("https://not.mapped.com", base_uri) self.assertEqual("./not.mapped.com", destination) desmap.set_root_folder("foo/bar") uri = "http://long.name.com/path/to/resource.xml" base_uri, destination = desmap.find_destination(uri) self.assertEqual("http://long.name.com", base_uri) self.assertEqual("foo/bar/destination1", destination)
def setUp(self): DestinationMap.__set_map_filename__("test-files/desmap.txt") DestinationMap().__drop__()
def setUp(self): Config.__set_config_filename__("test-files/config.txt") Config().__drop__() DestinationMap.__set_map_filename__("test-files/desmap.txt") DestinationMap().__drop__() des.desclient.reset_instance()