def __create_resourcelist__(src, checksum=True): """ Create a resourcelist for the source denominated by src. :param src: 's1', 's2', 'foo' etc. :param checksum: should checksums be added to the xml. :return: resync.resource_list.ResourceList """ abs_path = os.path.dirname(os.path.abspath(__name__)) data = [] path = os.path.join(abs_path, "rs/source", src, "files") for root, directories, filenames in os.walk(path): for filename in filenames: data.append(os.path.join(root,filename)) paths = ",".join(data) # create a resourcelist from the files in test/rs/source/{src}/files client = Client(checksum=checksum) prefix = "http://localhost:8000/rs/source/" + src + "/files" resourcedir = os.path.join(abs_path, "rs/source", src, "files") args = [prefix, resourcedir] client.set_mappings(args) rl = client.build_resource_list(paths=paths, set_path=True) return rl
def __create_resourcelist__(src, checksum=True): """ Create a resourcelist for the source denominated by src. :param src: 's1', 's2', 'foo' etc. :param checksum: should checksums be added to the xml. :return: resync.resource_list.ResourceList """ abs_path = os.path.dirname(os.path.abspath(__name__)) data = [] path = os.path.join(abs_path, "rs/source", src, "files") for root, directories, filenames in os.walk(path): for filename in filenames: data.append(os.path.join(root, filename)) paths = ",".join(data) # create a resourcelist from the files in test/rs/source/{src}/files client = Client(checksum=checksum) prefix = "http://localhost:8000/rs/source/" + src + "/files" resourcedir = os.path.join(abs_path, "rs/source", src, "files") args = [prefix, resourcedir] client.set_mappings(args) rl = client.build_resource_list(paths=paths, set_path=True) return rl
def __create_changelist__(src, checksum=True): """ Create a changelist xml for the source denominated by src. :param src: 's1', 's2' etc. :param checksum: should checksums be added to the xml. :return: None """ abs_path = os.path.dirname(os.path.abspath(__name__)) data = [] path = os.path.join(abs_path, "rs/source", src, "files") for root, directories, filenames in os.walk(path): for filename in filenames: data.append(os.path.join(root, filename)) paths = ",".join(data) outfile = os.path.join(abs_path, "rs/source", src, "changelist.xml") ref_sitemap = pathlib.Path(os.path.join(abs_path, "rs/source", src, "resourcelist.xml")).as_uri() # create a changelist from the files in test/rs/source/{src}/files based on ^that client = Client(checksum=checksum) prefix = "http://localhost:8000/rs/source/" + src + "/files" resourcedir = os.path.join(abs_path, "rs/source", src, "files") args = [prefix, resourcedir] client.set_mappings(args) client.write_change_list(paths=paths, outfile=outfile, ref_sitemap=ref_sitemap)
def test06_write_capability_list(self): c = Client() with capture_stdout() as capturer: c.write_capability_list( { 'a':'uri_a', 'b':'uri_b' } ) self.assertTrue( re.search(r'<urlset ',capturer.result) ) self.assertTrue( re.search(r'<rs:md capability="capabilitylist" />',capturer.result) ) self.assertTrue( re.search(r'<url><loc>uri_a</loc><rs:md capability="a"',capturer.result) ) self.assertTrue( re.search(r'<url><loc>uri_b</loc><rs:md capability="b"',capturer.result) )
def test48_write_dump_if_requested(self): c = Client() # no dump file self.assertFalse( c.write_dump_if_requested( ChangeList(), None ) ) # with dump file with capture_stdout() as capturer: c.write_dump_if_requested(ChangeList(),'/tmp/a_file') self.assertTrue( re.search(r'FIXME', capturer.result) )
def test1_make_inventory_empty(self): c = Client() # No mapping is error self.assertRaises( TypeError, c.inventory ) # Supply empty mapping c.set_mappings( [] ) i = c.inventory self.assertEqual( len(i), 0 )
def test48_write_dump_if_requested(self): c = Client() # no dump file self.assertFalse(c.write_dump_if_requested(ChangeList(), None)) # with dump file with capture_stdout() as capturer: c.write_dump_if_requested(ChangeList(), '/tmp/a_file') self.assertTrue(re.search(r'FIXME', capturer.result))
def test07_write_source_description(self): c = Client() with capture_stdout() as capturer: c.write_source_description( [ 'a','b','c' ] ) #print capturer.result self.assertTrue( re.search(r'<urlset ',capturer.result) ) self.assertTrue( re.search(r'<rs:md capability="description" />',capturer.result) ) self.assertTrue( re.search(r'<url><loc>a</loc><rs:md capability="capabilitylist" /></url>',capturer.result) ) self.assertTrue( re.search(r'<url><loc>b</loc><rs:md capability="capabilitylist" /></url>',capturer.result) )
def test07_write_capability_list_index(self): c = Client() with capture_stdout() as capturer: c.write_capability_list_index( [ 'a','b','c' ] ) self.assertTrue( re.search(r'<sitemapindex ',capturer.result) ) self.assertTrue( re.search(r'<rs:md capability="capabilitylist" modified=',capturer.result) ) #print capturer.result self.assertTrue( re.search(r'<sitemap><loc>a</loc></sitemap>',capturer.result) ) self.assertTrue( re.search(r'<sitemap><loc>b</loc></sitemap>',capturer.result) )
def test04_read_resource_list(self): c = Client() rl = c.read_resource_list('tests/testdata/client/dir1/resourcelist.xml') self.assertEqual(len(rl), 3) rl = c.read_resource_list('file:tests/testdata/client/dir1/resourcelist.xml') self.assertEqual(len(rl), 3) self.assertEqual(len(rl), 3) self.assertRaises(ClientError, c.read_resource_list, 'file://tests/testdata/client/dir1/resourcelist.xml') self.assertRaises(ClientError, c.read_resource_list, 'file:///tests/testdata/client/dir1/resourcelist.xml') self.assertRaises(ClientError, c.read_resource_list, 'DOES_NOT_EXIST')
def test41_write_resource_list_path(self): c = Client() c.set_mappings( ['http://example.org/','resync/test/testdata/'] ) # with an explicit paths setting only the specified paths will be included with capture_stdout() as capturer: c.write_resource_list(paths='resync/test/testdata/dir1') sys.stderr.write(capturer.result) self.assertTrue( re.search(r'<rs:md at="\S+" capability="resourcelist"', capturer.result ) ) self.assertTrue( re.search(r'<url><loc>http://example.org/dir1/file_a</loc>', capturer.result ) ) self.assertTrue( re.search(r'<url><loc>http://example.org/dir1/file_b</loc>', capturer.result ) ) self.assertFalse( re.search(r'<url><loc>http://example.org/dir2/file_x</loc>', capturer.result ) )
def test40_write_resource_list_mappings(self): c = Client() c.set_mappings( ['http://example.org/','resync/test/testdata/'] ) # with no explicit paths seting the mapping will be used with capture_stdout() as capturer: c.write_resource_list() #sys.stderr.write(capturer.result) self.assertTrue( re.search(r'<rs:md at="\S+" capability="resourcelist"', capturer.result ) ) self.assertTrue( re.search(r'<url><loc>http://example.org/dir1/file_a</loc>', capturer.result ) ) self.assertTrue( re.search(r'<url><loc>http://example.org/dir1/file_b</loc>', capturer.result ) ) self.assertTrue( re.search(r'<url><loc>http://example.org/dir2/file_x</loc>', capturer.result ) )
def test47_write_source_description(self): c = Client() # simple case to STDOUT with capture_stdout() as capturer: c.write_source_description(['a', 'b', 'c']) self.assertTrue(re.search(r'<urlset ', capturer.result)) self.assertTrue( re.search(r'<rs:md capability="description" />', capturer.result)) self.assertTrue( re.search( r'<url><loc>a</loc><rs:md capability="capabilitylist" /></url>', capturer.result)) self.assertTrue( re.search( r'<url><loc>b</loc><rs:md capability="capabilitylist" /></url>', capturer.result)) # more complex case to STDOUT with capture_stdout() as capturer: c.write_source_description(capability_lists=['http://a.b/'], links=[{ 'rel': 'c', 'href': 'd' }]) self.assertTrue(re.search(r'http://a.b/', capturer.result)) # to file (just check that something is written) outfile = os.path.join(self.tmpdir, 'sd_out.xml') c.write_source_description(capability_lists=['http://a.b/'], outfile=outfile, links=[{ 'rel': 'c', 'href': 'd' }]) self.assertTrue(os.path.getsize(outfile) > 100)
def test06_write_capability_list(self): c = Client() with capture_stdout() as capturer: c.write_capability_list({'a': 'uri_a', 'b': 'uri_b'}) self.assertTrue(re.search(r'<urlset ', capturer.result)) self.assertTrue( re.search(r'<rs:md capability="capabilitylist" />', capturer.result)) self.assertTrue( re.search(r'<url><loc>uri_a</loc><rs:md capability="a"', capturer.result)) self.assertTrue( re.search(r'<url><loc>uri_b</loc><rs:md capability="b"', capturer.result))
def test46_write_capability_list(self): c = Client() caps = { 'a':'uri_a', 'b':'uri_b' } # simple case to STDOUT with capture_stdout() as capturer: c.write_capability_list( caps ) self.assertTrue( re.search(r'<urlset ',capturer.result) ) self.assertTrue( re.search(r'<rs:md capability="capabilitylist" />',capturer.result) ) self.assertTrue( re.search(r'<url><loc>uri_a</loc><rs:md capability="a"',capturer.result) ) self.assertTrue( re.search(r'<url><loc>uri_b</loc><rs:md capability="b"',capturer.result) ) # to file (just check that something is written) outfile = os.path.join(self.tmpdir,'caps_out.xml') c.write_capability_list(capabilities=caps, outfile=outfile) self.assertTrue( os.path.getsize(outfile)>100 )
def test04_read_resource_list(self): c = Client() rl = c.read_resource_list( 'tests/testdata/client/dir1/resourcelist.xml') self.assertEqual(len(rl), 3) rl = c.read_resource_list( 'file:tests/testdata/client/dir1/resourcelist.xml') self.assertEqual(len(rl), 3) self.assertEqual(len(rl), 3) self.assertRaises( ClientError, c.read_resource_list, 'file://tests/testdata/client/dir1/resourcelist.xml') self.assertRaises( ClientError, c.read_resource_list, 'file:///tests/testdata/client/dir1/resourcelist.xml') self.assertRaises(ClientError, c.read_resource_list, 'DOES_NOT_EXIST')
def test07_write_source_description(self): c = Client() with capture_stdout() as capturer: c.write_source_description(['a', 'b', 'c']) #print capturer.result self.assertTrue(re.search(r'<urlset ', capturer.result)) self.assertTrue( re.search(r'<rs:md capability="description" />', capturer.result)) self.assertTrue( re.search( r'<url><loc>a</loc><rs:md capability="capabilitylist" /></url>', capturer.result)) self.assertTrue( re.search( r'<url><loc>b</loc><rs:md capability="capabilitylist" /></url>', capturer.result))
def test1_make_inventory_empty(self): c = Client() # No mapping is error # def wrap_inventory_property_call(c): # do this because assertRaises( ClientFatalError, c.inventory ) doesn't work return (c.inventory) self.assertRaises(ClientFatalError, wrap_inventory_property_call, c)
def test45_write_change_list(self): c = Client() ex1 = 'tests/testdata/examples_from_spec/resourcesync_ex_1.xml' with capture_stdout() as capturer: c.write_change_list(ref_sitemap=ex1, newref_sitemap=ex1) self.assertTrue( re.search(r'<rs:md capability="changelist"', capturer.result) ) # compare ex1 with testdata on disk c.set_mappings( ['http://example.org/','tests/testdata/'] ) with capture_stdout() as capturer: c.write_change_list(ref_sitemap=ex1, paths='tests/testdata/dir1') self.assertTrue( re.search(r'<rs:md capability="changelist"', capturer.result) ) self.assertTrue( re.search(r'<url><loc>http://example.com/res1</loc><rs:md change="deleted" /></url>', capturer.result) ) # to file outfile = os.path.join(self.tmpdir,'cl_out.xml') c.write_change_list(ref_sitemap=ex1, newref_sitemap=ex1, outfile=outfile) self.assertTrue( os.path.getsize(outfile)>100 )
def sync_audit(map, counter): """Run resync audit.""" client = Client() # ignore fail to continue running, log later client.ignore_failures = True client.set_mappings(map) # init_logging(verbose=True) src_resource_list = client.find_resource_list() rlb = ResourceListBuilder(mapper=client.mapper) dst_resource_list = rlb.from_disk() # Compare these resource lists respecting any comparison options (same, updated, deleted, created) = dst_resource_list.compare(src_resource_list) result = dict(created=[], updated=[], deleted=[]) for item in created: record_id = item.uri.rsplit('/', 1)[1] result['created'].append(record_id) for item in updated: record_id = item.uri.rsplit('/', 1)[1] result['updated'].append(record_id) for item in deleted: record_id = item.uri.rsplit('/', 1)[1] result['deleted'].append(record_id) update_counter(counter, result) return dict(same=len(same), updated=len(updated), deleted=len(deleted), created=len(created))
def __create_changelist__(src, checksum=True): """ Create a changelist xml for the source denominated by src. :param src: 's1', 's2' etc. :param checksum: should checksums be added to the xml. :return: None """ abs_path = os.path.dirname(os.path.abspath(__name__)) data = [] path = os.path.join(abs_path, "rs/source", src, "files") for root, directories, filenames in os.walk(path): for filename in filenames: data.append(os.path.join(root, filename)) paths = ",".join(data) outfile = os.path.join(abs_path, "rs/source", src, "changelist.xml") ref_sitemap = pathlib.Path( os.path.join(abs_path, "rs/source", src, "resourcelist.xml")).as_uri() # create a changelist from the files in test/rs/source/{src}/files based on ^that client = Client(checksum=checksum) prefix = "http://localhost:8000/rs/source/" + src + "/files" resourcedir = os.path.join(abs_path, "rs/source", src, "files") args = [prefix, resourcedir] client.set_mappings(args) client.write_change_list(paths=paths, outfile=outfile, ref_sitemap=ref_sitemap)
def test45_write_change_list(self): c = Client() ex1 = 'tests/testdata/examples_from_spec/resourcesync_ex_1.xml' with capture_stdout() as capturer: c.write_change_list(ref_sitemap=ex1, newref_sitemap=ex1) self.assertTrue( re.search(r'<rs:md capability="changelist"', capturer.result)) # compare ex1 with testdata on disk c.set_mappings(['http://example.org/', 'tests/testdata/']) with capture_stdout() as capturer: c.write_change_list(ref_sitemap=ex1, paths='tests/testdata/dir1') self.assertTrue( re.search(r'<rs:md capability="changelist"', capturer.result)) self.assertTrue( re.search( r'<url><loc>http://example.com/res1</loc><rs:md change="deleted" /></url>', capturer.result)) # to file outfile = os.path.join(self.tmpdir, 'cl_out.xml') c.write_change_list(ref_sitemap=ex1, newref_sitemap=ex1, outfile=outfile) self.assertTrue(os.path.getsize(outfile) > 100)
def test10_baseline_or_audit(self): # FIXME - this is the guts of the client, tough to test, need to work through more cases... c = Client() dst = os.path.join(self.tmpdir, 'dst_dir1') c.set_mappings(['file:tests/testdata/client/dir1', dst]) # audit with empty dst, should say 3 to create with LogCapture() as lc: c.baseline_or_audit(audit_only=True) self.assertTrue( re.match(r'Status:\s+NOT IN SYNC.*to create=3', lc.records[-2].msg)) self.assertEqual(lc.records[-1].msg, 'Completed audit') # now do the sync with LogCapture() as lc: c.baseline_or_audit() self.assertTrue( re.match(r'Status:\s+SYNCED.*created=3', lc.records[-2].msg)) self.assertEqual(lc.records[-1].msg, 'Completed baseline sync')
def test10_baseline_or_audit(self): # FIXME - this is the guts of the client, tough to test, need to work through more cases... c = Client() dst = os.path.join(self.tmpdir,'dst_dir1') c.set_mappings( ['file:tests/testdata/client/dir1',dst] ) # audit with empty dst, should say 3 to create with LogCapture() as lc: c.baseline_or_audit(audit_only=True) self.assertTrue( re.match(r'Status:\s+NOT IN SYNC.*to create=3', lc.records[-2].msg) ) self.assertEqual( lc.records[-1].msg, 'Completed audit' ) # now do the sync with LogCapture() as lc: c.baseline_or_audit() self.assertTrue( re.match(r'Status:\s+SYNCED.*created=3', lc.records[-2].msg) ) self.assertEqual( lc.records[-1].msg, 'Completed baseline sync' )
def test47_write_source_description(self): c = Client() # simple case to STDOUT with capture_stdout() as capturer: c.write_source_description( [ 'a','b','c' ] ) self.assertTrue( re.search(r'<urlset ',capturer.result) ) self.assertTrue( re.search(r'<rs:md capability="description" />',capturer.result) ) self.assertTrue( re.search(r'<url><loc>a</loc><rs:md capability="capabilitylist" /></url>',capturer.result) ) self.assertTrue( re.search(r'<url><loc>b</loc><rs:md capability="capabilitylist" /></url>',capturer.result) ) # more complex case to STDOUT with capture_stdout() as capturer: c.write_source_description(capability_lists=['http://a.b/'], links=[{'rel':'c','href':'d'}]) self.assertTrue( re.search(r'http://a.b/', capturer.result ) ) # to file (just check that something is written) outfile = os.path.join(self.tmpdir,'sd_out.xml') c.write_source_description(capability_lists=['http://a.b/'], outfile=outfile, links=[{'rel':'c','href':'d'}]) self.assertTrue( os.path.getsize(outfile)>100 )
def test49_read_reference_resource_list(self): c = Client() with capture_stdout() as capturer: rl = c.read_reference_resource_list('tests/testdata/examples_from_spec/resourcesync_ex_1.xml') self.assertEqual( len(rl), 2 ) self.assertEqual( '', capturer.result ) c.verbose = True with capture_stdout() as capturer: rl = c.read_reference_resource_list('tests/testdata/examples_from_spec/resourcesync_ex_1.xml') self.assertEqual( len(rl), 2 ) self.assertTrue( re.search(r'http://example.com/res2', capturer.result) ) c.verbose = True c.max_sitemap_entries = 1 with capture_stdout() as capturer: rl = c.read_reference_resource_list('tests/testdata/examples_from_spec/resourcesync_ex_1.xml') self.assertEqual( len(rl), 2 ) self.assertTrue( re.search(r'http://example.com/res1', capturer.result) ) self.assertTrue( re.search(r'Showing first 1 entries', capturer.result) ) self.assertFalse( re.search(r'http://example.com/res2', capturer.result) )
def test49_read_reference_resource_list(self): c = Client() with capture_stdout() as capturer: rl = c.read_reference_resource_list( 'tests/testdata/examples_from_spec/resourcesync_ex_1.xml') self.assertEqual(len(rl), 2) self.assertEqual('', capturer.result) c.verbose = True with capture_stdout() as capturer: rl = c.read_reference_resource_list( 'tests/testdata/examples_from_spec/resourcesync_ex_1.xml') self.assertEqual(len(rl), 2) self.assertTrue(re.search(r'http://example.com/res2', capturer.result)) c.verbose = True c.max_sitemap_entries = 1 with capture_stdout() as capturer: rl = c.read_reference_resource_list( 'tests/testdata/examples_from_spec/resourcesync_ex_1.xml') self.assertEqual(len(rl), 2) self.assertTrue(re.search(r'http://example.com/res1', capturer.result)) self.assertTrue(re.search(r'Showing first 1 entries', capturer.result)) self.assertFalse(re.search(r'http://example.com/res2', capturer.result))
def test41_write_resource_list_path(self): c = Client() c.set_mappings(['http://example.org/', 'resync/test/testdata/']) # with an explicit paths setting only the specified paths will be included with capture_stdout() as capturer: c.write_resource_list(paths='resync/test/testdata/dir1') sys.stderr.write(capturer.result) self.assertTrue( re.search(r'<rs:md at="\S+" capability="resourcelist"', capturer.result)) self.assertTrue( re.search(r'<url><loc>http://example.org/dir1/file_a</loc>', capturer.result)) self.assertTrue( re.search(r'<url><loc>http://example.org/dir1/file_b</loc>', capturer.result)) self.assertFalse( re.search(r'<url><loc>http://example.org/dir2/file_x</loc>', capturer.result))
def test40_write_resource_list_mappings(self): c = Client() c.set_mappings(['http://example.org/', 'resync/test/testdata/']) # with no explicit paths seting the mapping will be used with capture_stdout() as capturer: c.write_resource_list() #sys.stderr.write(capturer.result) self.assertTrue( re.search(r'<rs:md at="\S+" capability="resourcelist"', capturer.result)) self.assertTrue( re.search(r'<url><loc>http://example.org/dir1/file_a</loc>', capturer.result)) self.assertTrue( re.search(r'<url><loc>http://example.org/dir1/file_b</loc>', capturer.result)) self.assertTrue( re.search(r'<url><loc>http://example.org/dir2/file_x</loc>', capturer.result))
def test46_write_capability_list(self): c = Client() caps = {'a': 'uri_a', 'b': 'uri_b'} # simple case to STDOUT with capture_stdout() as capturer: c.write_capability_list(caps) self.assertTrue(re.search(r'<urlset ', capturer.result)) self.assertTrue( re.search(r'<rs:md capability="capabilitylist" />', capturer.result)) self.assertTrue( re.search(r'<url><loc>uri_a</loc><rs:md capability="a"', capturer.result)) self.assertTrue( re.search(r'<url><loc>uri_b</loc><rs:md capability="b"', capturer.result)) # to file (just check that something is written) outfile = os.path.join(self.tmpdir, 'caps_out.xml') c.write_capability_list(capabilities=caps, outfile=outfile) self.assertTrue(os.path.getsize(outfile) > 100)
def test20_parse_document(self): # Key property of the parse_document() method is that it parses the # document and identifies its type c = Client() with capture_stdout() as capturer: c.sitemap_name='resync/test/testdata/examples_from_spec/resourcesync_ex_1.xml' c.parse_document() self.assertTrue( re.search(r'Parsed resourcelist document with 2 entries',capturer.result) ) with capture_stdout() as capturer: c.sitemap_name='resync/test/testdata/examples_from_spec/resourcesync_ex_17.xml' c.parse_document() self.assertTrue( re.search(r'Parsed resourcedump document with 3 entries',capturer.result) ) with capture_stdout() as capturer: c.sitemap_name='resync/test/testdata/examples_from_spec/resourcesync_ex_19.xml' c.parse_document() self.assertTrue( re.search(r'Parsed changelist document with 4 entries',capturer.result) ) with capture_stdout() as capturer: c.sitemap_name='resync/test/testdata/examples_from_spec/resourcesync_ex_22.xml' c.parse_document() self.assertTrue( re.search(r'Parsed changedump document with 3 entries',capturer.result) )
def test19_delete_resource(self): c = Client() resource = Resource(uri='http://example.org/1') filename = os.path.join(self.tmpdir, 'resource1') c.last_timestamp = 5 # no delete, no timestamp update with LogCapture() as lc: c.logger = logging.getLogger('resync.client') n = c.delete_resource(resource, filename) self.assertEqual(n, 0) self.assertEqual( lc.records[-1].msg, 'nodelete: would delete http://example.org/1 (--delete to enable)' ) self.assertEqual(c.last_timestamp, 5) # no delete but timestamp update resource.timestamp = 10 with LogCapture() as lc: c.logger = logging.getLogger('resync.client') n = c.delete_resource(resource, filename) self.assertEqual(n, 0) self.assertEqual( lc.records[-1].msg, 'nodelete: would delete http://example.org/1 (--delete to enable)' ) self.assertEqual(c.last_timestamp, 10) # allow delete but dryrun with LogCapture() as lc: c.dryrun = True c.logger = logging.getLogger('resync.client') n = c.delete_resource(resource, filename, allow_deletion=True) self.assertEqual(n, 0) self.assertTrue(lc.records[-1].msg.startswith( 'dryrun: would delete http://example.org/1')) c.dryrun = False # allow delete but no resource present with LogCapture() as lc: c.logger = logging.getLogger('resync.client') n = c.delete_resource(resource, filename, allow_deletion=True) self.assertEqual(n, 0) self.assertTrue(lc.records[-1].msg.startswith( 'Failed to DELETE http://example.org/1')) # successful deletion, first make file... with open(filename, 'w') as fh: fh.write('delete me') fh.close() with LogCapture() as lc: c.logger = logging.getLogger('resync.client') n = c.delete_resource(resource, filename, allow_deletion=True) self.assertEqual(n, 1) self.assertTrue(lc.records[-1].msg.startswith('Event: {')) self.assertTrue(lc.records[-2].msg.startswith( 'deleted: http://example.org/1 ->'))
def test05_find_resource_list(self): c = Client() # Filesystem tests c.set_mappings(['tests/testdata/find/find1', 'xxx']) self.assertEqual(c.find_resource_list().up, 'find1') c.set_mappings(['tests/testdata/find/find2', 'xxx']) self.assertEqual(c.find_resource_list().up, 'find2') c.set_mappings(['tests/testdata/find/find3', 'xxx']) self.assertEqual(c.find_resource_list().up, 'find3') # Tests requiring a server with webserver('tests/testdata/find', 'localhost', 9999): c.set_mappings(['http://localhost:9999/find1', 'xxx']) self.assertEqual(c.find_resource_list().up, 'find1') c.set_mappings(['http://localhost:9999/find2', 'xxx']) self.assertEqual(c.find_resource_list().up, 'find2') c.set_mappings(['http://localhost:9999/find3', 'xxx']) self.assertEqual(c.find_resource_list().up, 'find3') with webserver('tests/testdata/find/find1', 'localhost', 9999): c.set_mappings(['http://localhost:9999/data', 'xxx']) self.assertEqual(c.find_resource_list().up, 'find1') with webserver('tests/testdata/find/find3', 'localhost', 9999): c.set_mappings(['http://localhost:9999/data/data1', 'xxx']) self.assertEqual(c.find_resource_list().up, 'find3')
def test2_bad_source_uri(self): c = Client(logger=logging.getLogger()) self.assertRaises(ClientFatalError, c.sync_or_audit) c.set_mappings(['http://example.org/bbb', '/tmp/this_does_not_exist']) self.assertRaises(ClientFatalError, c.sync_or_audit)
def test18_update_resource(self): c = Client() resource = Resource(uri='http://example.org/dir/2') filename = os.path.join(self.tmpdir,'dir/resource2') # dryrun with LogCapture() as lc: c.dryrun = True c.logger = logging.getLogger('resync.client') n = c.update_resource( resource, filename ) self.assertEqual( n, 0 ) self.assertTrue( lc.records[-1].msg.startswith('dryrun: would GET http://example.org/dir/2 ') ) c.dryrun = False # get from file uri that does not exist resource = Resource(uri='file:tests/testdata/i_do_not_exist') self.assertRaises( ClientFatalError, c.update_resource, resource, filename ) # get from file uri that does not exist but with c.ignore_failures to log resource = Resource(uri='file:tests/testdata/i_do_not_exist') with LogCapture() as lc: c.logger = logging.getLogger('resync.client') c.ignore_failures = True n = c.update_resource( resource, filename ) self.assertEqual( n, 0 ) self.assertTrue( lc.records[-1].msg.startswith('Failed to GET file:tests/testdata/i_do_not_exist ') ) # get from file uri resource = Resource(uri='file:tests/testdata/examples_from_spec/resourcesync_ex_1.xml', length=355, md5='abc', timestamp=10) c.last_timestamp = 0 with LogCapture() as lc: c.logger = logging.getLogger('resync.client') n = c.update_resource( resource, filename ) self.assertEqual( n, 1 ) self.assertTrue( lc.records[-1].msg.startswith('Event: {') ) # get from file uri with length and md5 warnings resource = Resource(uri='file:tests/testdata/examples_from_spec/resourcesync_ex_1.xml', length=111, md5='abc', timestamp=10) c.last_timestamp = 0 with LogCapture() as lc: c.logger = logging.getLogger('resync.client') c.checksum = True n = c.update_resource( resource, filename ) self.assertEqual( n, 1 ) self.assertTrue( lc.records[-1].msg.startswith('MD5 mismatch ') ) self.assertTrue( lc.records[-2].msg.startswith('Downloaded size for ') ) self.assertTrue( lc.records[-3].msg.startswith('Event: {') )
def test41_write_resource_list_path(self): c = Client() c.set_mappings(['http://example.org/', 'tests/testdata/']) links = [{'rel': 'uri_c', 'href': 'uri_d'}] # with an explicit paths setting only the specified paths will be # included with capture_stdout() as capturer: c.write_resource_list(paths='tests/testdata/dir1', links=links) self.assertTrue( re.search(r'<rs:md at="\S+" capability="resourcelist"', capturer.result)) self.assertTrue( re.search(r'<url><loc>http://example.org/dir1/file_a</loc>', capturer.result)) self.assertTrue( re.search(r'<url><loc>http://example.org/dir1/file_b</loc>', capturer.result)) self.assertFalse( re.search(r'<url><loc>http://example.org/dir2/file_x</loc>', capturer.result)) # check link present self.assertTrue(re.search(r'rel="uri_c"', capturer.result)) self.assertTrue(re.search(r'href="uri_d"', capturer.result)) # Travis CI does not preserve timestamps from github so test here for the file # size but not the datestamp # self.assertTrue( re.search(r'<url><loc>http://example.org/dir1/file_a</loc><lastmod>[\w\-:]+</lastmod><rs:md length="20" /></url>', capturer.result ) ) # self.assertTrue( re.search(r'<url><loc>http://example.org/dir1/file_b</loc><lastmod>[\w\-:]+</lastmod><rs:md length="45" /></url>', capturer.result ) ) # to file outfile = os.path.join(self.tmpdir, 'rl_out.xml') c.write_resource_list(paths='tests/testdata/dir1', outfile=outfile) self.assertTrue(os.path.getsize(outfile) > 100) # dump instead (default file) c.default_resource_dump = os.path.join(self.tmpdir, 'rl_out_dump_def') outfile = c.default_resource_dump + '00000.zip' self.assertFalse(os.path.exists(outfile)) c.write_resource_list(paths='tests/testdata/dir1', dump=True) self.assertTrue(os.path.getsize(outfile) > 100) # (specific file) outbase = os.path.join(self.tmpdir, 'rl_out_dump') outfile = outbase + '00000.zip' self.assertFalse(os.path.exists(outfile)) c.write_resource_list(paths='tests/testdata/dir1', dump=True, outfile=outbase) self.assertTrue(os.path.getsize(outfile) > 100)
def test19_delete_resource(self): c = Client() resource = Resource(uri='http://example.org/1') filename = os.path.join(self.tmpdir,'resource1') c.last_timestamp = 5 # no delete, no timestamp update with LogCapture() as lc: c.logger = logging.getLogger('resync.client') n = c.delete_resource( resource, filename ) self.assertEqual( n, 0 ) self.assertEqual( lc.records[-1].msg, 'nodelete: would delete http://example.org/1 (--delete to enable)' ) self.assertEqual( c.last_timestamp, 5 ) # no delete but timestamp update resource.timestamp = 10 with LogCapture() as lc: c.logger = logging.getLogger('resync.client') n = c.delete_resource( resource, filename ) self.assertEqual( n, 0 ) self.assertEqual( lc.records[-1].msg, 'nodelete: would delete http://example.org/1 (--delete to enable)' ) self.assertEqual( c.last_timestamp, 10 ) # allow delete but dryrun with LogCapture() as lc: c.dryrun = True c.logger = logging.getLogger('resync.client') n = c.delete_resource( resource, filename, allow_deletion=True ) self.assertEqual( n, 0 ) self.assertTrue( lc.records[-1].msg.startswith('dryrun: would delete http://example.org/1') ) c.dryrun = False # allow delete but no resource present with LogCapture() as lc: c.logger = logging.getLogger('resync.client') n = c.delete_resource( resource, filename, allow_deletion=True ) self.assertEqual( n, 0 ) self.assertTrue( lc.records[-1].msg.startswith('Failed to DELETE http://example.org/1') ) # successful deletion, first make file... with open(filename, 'w') as fh: fh.write('delete me') fh.close() with LogCapture() as lc: c.logger = logging.getLogger('resync.client') n = c.delete_resource( resource, filename, allow_deletion=True ) self.assertEqual( n, 1 ) self.assertTrue( lc.records[-1].msg.startswith('Event: {') ) self.assertTrue( lc.records[-2].msg.startswith('deleted: http://example.org/1 ->') )
def test03_sitemap_uri(self): c = Client() c.set_mappings(['http://example.org/c', '/tmp/not_there_at_all']) self.assertEqual(c.sitemap_uri('abcd1'), 'http://example.org/c/abcd1') self.assertEqual(c.sitemap_uri('/abcd2'), '/abcd2') self.assertEqual(c.sitemap_uri('scheme:/abcd3'), 'scheme:/abcd3')
def test50_log_status(self): c = Client() with LogCapture() as lc: c.logger = logging.getLogger('resync.client') c.log_status() self.assertEqual( lc.records[-1].msg, 'Status: IN SYNC (created=0, updated=0, deleted=0)') c.log_status(created=123, updated=456, deleted=789) self.assertEqual( lc.records[-1].msg, 'Status: IN SYNC (created=123, updated=456, deleted=789)' ) c.log_status(incremental=True) self.assertEqual( lc.records[-1].msg, 'Status: NO CHANGES (created=0, updated=0, deleted=0)') c.log_status(audit=True) self.assertEqual( lc.records[-1].msg, 'Status: IN SYNC (to create=0, to update=0, to delete=0)' ) c.log_status(in_sync=False, audit=True) self.assertEqual( lc.records[-1].msg, 'Status: NOT IN SYNC (to create=0, to update=0, to delete=0)' ) c.log_status(in_sync=False, audit=False, to_delete=1) self.assertEqual( lc.records[-1].msg, 'Status: PART SYNCED (created=0, updated=0, to delete (--delete)=1)' ) c.log_status(in_sync=False, audit=False, to_delete=1, incremental=1) self.assertEqual( lc.records[-1].msg, 'Status: PART APPLIED (created=0, updated=0, to delete (--delete)=1)' ) c.log_status(in_sync=False) self.assertEqual( lc.records[-1].msg, 'Status: SYNCED (created=0, updated=0, deleted=0)')
def test20_parse_document(self): # Key property of the parse_document() method is that it parses the # document and identifies its type c = Client() with capture_stdout() as capturer: c.sitemap_name = 'tests/testdata/examples_from_spec/resourcesync_ex_1.xml' c.parse_document() self.assertTrue( re.search(r'Parsed resourcelist document with 2 entries', capturer.result)) with capture_stdout() as capturer: c.sitemap_name = 'tests/testdata/examples_from_spec/resourcesync_ex_17.xml' c.parse_document() self.assertTrue( re.search(r'Parsed resourcedump document with 3 entries', capturer.result)) with capture_stdout() as capturer: c.sitemap_name = 'tests/testdata/examples_from_spec/resourcesync_ex_19.xml' c.parse_document() self.assertTrue( re.search(r'Parsed changelist document with 4 entries', capturer.result)) with capture_stdout() as capturer: c.sitemap_name = 'tests/testdata/examples_from_spec/resourcesync_ex_22.xml' c.parse_document() self.assertTrue( re.search(r'Parsed changedump document with 3 entries', capturer.result)) # Document that doesn't exist c.sitemap_name = '/does_not_exist' self.assertRaises(ClientFatalError, c.parse_document) # and verbose with truncation... with capture_stdout() as capturer: c.verbose = True c.sitemap_name = 'tests/testdata/examples_from_spec/resourcesync_ex_1.xml' c.max_sitemap_entries = 1 c.parse_document() self.assertTrue(re.search(r'Showing first 1 entries', capturer.result)) self.assertTrue(re.search(r'\[0\] ', capturer.result)) self.assertFalse(re.search(r'\[1\] ', capturer.result))
def test02_bad_source_uri(self): c = Client() self.assertRaises(ClientFatalError, c.baseline_or_audit) c.set_mappings(['http://example.org/bbb', '/tmp/this_does_not_exist']) self.assertRaises(ClientFatalError, c.baseline_or_audit)
def test41_write_resource_list_path(self): c = Client() c.set_mappings( ['http://example.org/','tests/testdata/'] ) links=[{'rel':'uri_c','href':'uri_d'}] # with an explicit paths setting only the specified paths will be included with capture_stdout() as capturer: c.write_resource_list(paths='tests/testdata/dir1', links=links) self.assertTrue( re.search(r'<rs:md at="\S+" capability="resourcelist"', capturer.result ) ) self.assertTrue( re.search(r'<url><loc>http://example.org/dir1/file_a</loc>', capturer.result ) ) self.assertTrue( re.search(r'<url><loc>http://example.org/dir1/file_b</loc>', capturer.result ) ) self.assertFalse( re.search(r'<url><loc>http://example.org/dir2/file_x</loc>', capturer.result ) ) # check link present self.assertTrue( re.search(r'rel="uri_c"', capturer.result ) ) self.assertTrue( re.search(r'href="uri_d"', capturer.result ) ) # Travis CI does not preserve timestamps from github so test here for the file # size but not the datestamp #self.assertTrue( re.search(r'<url><loc>http://example.org/dir1/file_a</loc><lastmod>[\w\-:]+</lastmod><rs:md length="20" /></url>', capturer.result ) ) #self.assertTrue( re.search(r'<url><loc>http://example.org/dir1/file_b</loc><lastmod>[\w\-:]+</lastmod><rs:md length="45" /></url>', capturer.result ) ) # to file outfile = os.path.join(self.tmpdir,'rl_out.xml') c.write_resource_list(paths='tests/testdata/dir1', outfile=outfile) self.assertTrue( os.path.getsize(outfile)>100 ) # dump instead (default file) c.default_resource_dump = os.path.join(self.tmpdir,'rl_out_dump_def') outfile = c.default_resource_dump+'00000.zip' self.assertFalse( os.path.exists(outfile) ) c.write_resource_list(paths='tests/testdata/dir1', dump=True) self.assertTrue( os.path.getsize(outfile)>100 ) # (specific file) outbase = os.path.join(self.tmpdir,'rl_out_dump') outfile = outbase+'00000.zip' self.assertFalse( os.path.exists(outfile) ) c.write_resource_list(paths='tests/testdata/dir1', dump=True, outfile=outbase) self.assertTrue( os.path.getsize(outfile)>100 )
def test20_parse_document(self): # Key property of the parse_document() method is that it parses the # document and identifies its type c = Client() with capture_stdout() as capturer: c.sitemap_name='tests/testdata/examples_from_spec/resourcesync_ex_1.xml' c.parse_document() self.assertTrue( re.search(r'Parsed resourcelist document with 2 entries',capturer.result) ) with capture_stdout() as capturer: c.sitemap_name='tests/testdata/examples_from_spec/resourcesync_ex_17.xml' c.parse_document() self.assertTrue( re.search(r'Parsed resourcedump document with 3 entries',capturer.result) ) with capture_stdout() as capturer: c.sitemap_name='tests/testdata/examples_from_spec/resourcesync_ex_19.xml' c.parse_document() self.assertTrue( re.search(r'Parsed changelist document with 4 entries',capturer.result) ) with capture_stdout() as capturer: c.sitemap_name='tests/testdata/examples_from_spec/resourcesync_ex_22.xml' c.parse_document() self.assertTrue( re.search(r'Parsed changedump document with 3 entries',capturer.result) ) # Document that doesn't exist c.sitemap_name='/does_not_exist' self.assertRaises( ClientFatalError, c.parse_document ) # and verbose with truncation... with capture_stdout() as capturer: c.verbose = True c.sitemap_name = 'tests/testdata/examples_from_spec/resourcesync_ex_1.xml' c.max_sitemap_entries = 1 c.parse_document() self.assertTrue( re.search(r'Showing first 1 entries', capturer.result ) ) self.assertTrue( re.search(r'\[0\] ', capturer.result ) ) self.assertFalse( re.search(r'\[1\] ', capturer.result ) )
def test2_bad_source_uri(self): c = Client() self.assertRaises( ClientFatalError, c.sync_or_audit, 'a', '/tmp/bbb' ) self.assertRaises( ClientFatalError, c.sync_or_audit, 'http://example.org/bbb', '/tmp/bbb' )
def test03_sitemap_uri(self): c = Client() c.set_mappings( ['http://example.org/c','/tmp/not_there_at_all'] ) self.assertEqual( c.sitemap_uri('abcd1'), 'http://example.org/c/abcd1' ) self.assertEqual( c.sitemap_uri('/abcd2'), '/abcd2' ) self.assertEqual( c.sitemap_uri('scheme:/abcd3'), 'scheme:/abcd3' )
def test20_parse_document(self): # Key property of the parse_document() method is that it parses the # document and identifies its type c = Client() with capture_stdout() as capturer: c.sitemap_name = 'resync/test/testdata/examples_from_spec/resourcesync_ex_1.xml' c.parse_document() self.assertTrue( re.search(r'Parsed resourcelist document with 2 entries', capturer.result)) with capture_stdout() as capturer: c.sitemap_name = 'resync/test/testdata/examples_from_spec/resourcesync_ex_17.xml' c.parse_document() self.assertTrue( re.search(r'Parsed resourcedump document with 3 entries', capturer.result)) with capture_stdout() as capturer: c.sitemap_name = 'resync/test/testdata/examples_from_spec/resourcesync_ex_19.xml' c.parse_document() self.assertTrue( re.search(r'Parsed changelist document with 4 entries', capturer.result)) with capture_stdout() as capturer: c.sitemap_name = 'resync/test/testdata/examples_from_spec/resourcesync_ex_22.xml' c.parse_document() self.assertTrue( re.search(r'Parsed changedump document with 3 entries', capturer.result))
def test50_log_status(self): c = Client() with LogCapture() as lc: c.logger = logging.getLogger('resync.client') c.log_status() self.assertEqual( lc.records[-1].msg, 'Status: IN SYNC (created=0, updated=0, deleted=0)' ) c.log_status(created=123,updated=456,deleted=789) self.assertEqual( lc.records[-1].msg, 'Status: IN SYNC (created=123, updated=456, deleted=789)' ) c.log_status(incremental=True) self.assertEqual( lc.records[-1].msg, 'Status: NO CHANGES (created=0, updated=0, deleted=0)' ) c.log_status(audit=True) self.assertEqual( lc.records[-1].msg, 'Status: IN SYNC (to create=0, to update=0, to delete=0)' ) c.log_status(in_sync=False, audit=True) self.assertEqual( lc.records[-1].msg, 'Status: NOT IN SYNC (to create=0, to update=0, to delete=0)' ) c.log_status(in_sync=False, audit=False, to_delete=1) self.assertEqual( lc.records[-1].msg, 'Status: PART SYNCED (created=0, updated=0, to delete (--delete)=1)' ) c.log_status(in_sync=False, audit=False, to_delete=1, incremental=1) self.assertEqual( lc.records[-1].msg, 'Status: PART APPLIED (created=0, updated=0, to delete (--delete)=1)' ) c.log_status(in_sync=False) self.assertEqual( lc.records[-1].msg, 'Status: SYNCED (created=0, updated=0, deleted=0)' )
def test2_bad_source_uri(self): c = Client(logger=logging.getLogger()) self.assertRaises( ClientFatalError, c.sync_or_audit ) c.set_mappings( ['http://example.org/bbb','/tmp/this_does_not_exist'] ) self.assertRaises( ClientFatalError, c.sync_or_audit )
def test18_update_resource(self): c = Client() resource = Resource(uri='http://example.org/dir/2') filename = os.path.join(self.tmpdir, 'dir/resource2') # dryrun with LogCapture() as lc: c.dryrun = True c.logger = logging.getLogger('resync.client') n = c.update_resource(resource, filename) self.assertEqual(n, 0) self.assertTrue(lc.records[-1].msg.startswith( 'dryrun: would GET http://example.org/dir/2 ')) c.dryrun = False # get from file uri that does not exist resource = Resource(uri='file:tests/testdata/i_do_not_exist') self.assertRaises(ClientFatalError, c.update_resource, resource, filename) # get from file uri that does not exist but with c.ignore_failures to # log resource = Resource(uri='file:tests/testdata/i_do_not_exist') with LogCapture() as lc: c.logger = logging.getLogger('resync.client') c.ignore_failures = True n = c.update_resource(resource, filename) self.assertEqual(n, 0) self.assertTrue(lc.records[-1].msg.startswith( 'Failed to GET file:tests/testdata/i_do_not_exist ')) # get from file uri resource = Resource( uri='file:tests/testdata/examples_from_spec/resourcesync_ex_1.xml', length=355, md5='abc', timestamp=10) c.last_timestamp = 0 with LogCapture() as lc: c.logger = logging.getLogger('resync.client') n = c.update_resource(resource, filename) self.assertEqual(n, 1) self.assertTrue(lc.records[-1].msg.startswith('Event: {')) # get from file uri with length and md5 warnings resource = Resource( uri='file:tests/testdata/examples_from_spec/resourcesync_ex_1.xml', length=111, md5='abc', timestamp=10) c.last_timestamp = 0 with LogCapture() as lc: c.logger = logging.getLogger('resync.client') c.hashes = set(['md5']) n = c.update_resource(resource, filename) self.assertEqual(n, 1) self.assertTrue(lc.records[-1].msg.startswith('MD5 mismatch ')) self.assertTrue( lc.records[-2].msg.startswith('Downloaded size for ')) self.assertTrue(lc.records[-3].msg.startswith('Event: {'))
def test2_bad_source_uri(self): c = Client() self.assertRaises( ClientFatalError, c.baseline_or_audit ) c.set_mappings( ['http://example.org/bbb','/tmp/this_does_not_exist'] ) self.assertRaises( ClientFatalError, c.baseline_or_audit )
def test01_make_resource_list_empty(self): c = Client() # No mapping is an error self.assertRaises(ClientFatalError, c.build_resource_list)