def __create_resourcelist__(src, checksum=True):
"""
Create a resourcelist for the source denominated by src.
:param src: 's1', 's2', 'foo' etc.
:param checksum: should checksums be added to the xml.
:return: resync.resource_list.ResourceList
"""
abs_path = os.path.dirname(os.path.abspath(__name__))
data = []
path = os.path.join(abs_path, "rs/source", src, "files")
for root, directories, filenames in os.walk(path):
for filename in filenames:
data.append(os.path.join(root,filename))
paths = ",".join(data)
# create a resourcelist from the files in test/rs/source/{src}/files
client = Client(checksum=checksum)
prefix = "http://localhost:8000/rs/source/" + src + "/files"
resourcedir = os.path.join(abs_path, "rs/source", src, "files")
args = [prefix, resourcedir]
client.set_mappings(args)
rl = client.build_resource_list(paths=paths, set_path=True)
return rl
def test41_write_resource_list_path(self):
c = Client()
c.set_mappings( ['http://example.org/','tests/testdata/'] )
links=[{'rel':'uri_c','href':'uri_d'}]
# with an explicit paths setting only the specified paths will be included
with capture_stdout() as capturer:
c.write_resource_list(paths='tests/testdata/dir1', links=links)
self.assertTrue( re.search(r'<rs:md at="\S+" capability="resourcelist"', capturer.result ) )
self.assertTrue( re.search(r'<url><loc>http://example.org/dir1/file_a</loc>', capturer.result ) )
self.assertTrue( re.search(r'<url><loc>http://example.org/dir1/file_b</loc>', capturer.result ) )
self.assertFalse( re.search(r'<url><loc>http://example.org/dir2/file_x</loc>', capturer.result ) )
# check link present
self.assertTrue( re.search(r'rel="uri_c"', capturer.result ) )
self.assertTrue( re.search(r'href="uri_d"', capturer.result ) )
# Travis CI does not preserve timestamps from github so test here for the file
# size but not the datestamp
#self.assertTrue( re.search(r'<url><loc>http://example.org/dir1/file_a</loc><lastmod>[\w\-:]+</lastmod><rs:md length="20" /></url>', capturer.result ) )
#self.assertTrue( re.search(r'<url><loc>http://example.org/dir1/file_b</loc><lastmod>[\w\-:]+</lastmod><rs:md length="45" /></url>', capturer.result ) )
# to file
outfile = os.path.join(self.tmpdir,'rl_out.xml')
c.write_resource_list(paths='tests/testdata/dir1', outfile=outfile)
self.assertTrue( os.path.getsize(outfile)>100 )
# dump instead (default file)
c.default_resource_dump = os.path.join(self.tmpdir,'rl_out_dump_def')
outfile = c.default_resource_dump+'00000.zip'
self.assertFalse( os.path.exists(outfile) )
c.write_resource_list(paths='tests/testdata/dir1', dump=True)
self.assertTrue( os.path.getsize(outfile)>100 )
# (specific file)
outbase = os.path.join(self.tmpdir,'rl_out_dump')
outfile = outbase+'00000.zip'
self.assertFalse( os.path.exists(outfile) )
c.write_resource_list(paths='tests/testdata/dir1', dump=True, outfile=outbase)
self.assertTrue( os.path.getsize(outfile)>100 )
def __create_resourcelist__(src, checksum=True):
"""
Create a resourcelist for the source denominated by src.
:param src: 's1', 's2', 'foo' etc.
:param checksum: should checksums be added to the xml.
:return: resync.resource_list.ResourceList
"""
abs_path = os.path.dirname(os.path.abspath(__name__))
data = []
path = os.path.join(abs_path, "rs/source", src, "files")
for root, directories, filenames in os.walk(path):
for filename in filenames:
data.append(os.path.join(root, filename))
paths = ",".join(data)
# create a resourcelist from the files in test/rs/source/{src}/files
client = Client(checksum=checksum)
prefix = "http://localhost:8000/rs/source/" + src + "/files"
resourcedir = os.path.join(abs_path, "rs/source", src, "files")
args = [prefix, resourcedir]
client.set_mappings(args)
rl = client.build_resource_list(paths=paths, set_path=True)
return rl
def __create_changelist__(src, checksum=True):
"""
Create a changelist xml for the source denominated by src.
:param src: 's1', 's2' etc.
:param checksum: should checksums be added to the xml.
:return: None
"""
abs_path = os.path.dirname(os.path.abspath(__name__))
data = []
path = os.path.join(abs_path, "rs/source", src, "files")
for root, directories, filenames in os.walk(path):
for filename in filenames:
data.append(os.path.join(root, filename))
paths = ",".join(data)
outfile = os.path.join(abs_path, "rs/source", src, "changelist.xml")
ref_sitemap = pathlib.Path(
os.path.join(abs_path, "rs/source", src, "resourcelist.xml")).as_uri()
# create a changelist from the files in test/rs/source/{src}/files based on ^that
client = Client(checksum=checksum)
prefix = "http://localhost:8000/rs/source/" + src + "/files"
resourcedir = os.path.join(abs_path, "rs/source", src, "files")
args = [prefix, resourcedir]
client.set_mappings(args)
client.write_change_list(paths=paths,
outfile=outfile,
ref_sitemap=ref_sitemap)
def __create_changelist__(src, checksum=True):
"""
Create a changelist xml for the source denominated by src.
:param src: 's1', 's2' etc.
:param checksum: should checksums be added to the xml.
:return: None
"""
abs_path = os.path.dirname(os.path.abspath(__name__))
data = []
path = os.path.join(abs_path, "rs/source", src, "files")
for root, directories, filenames in os.walk(path):
for filename in filenames:
data.append(os.path.join(root, filename))
paths = ",".join(data)
outfile = os.path.join(abs_path, "rs/source", src, "changelist.xml")
ref_sitemap = pathlib.Path(os.path.join(abs_path, "rs/source", src, "resourcelist.xml")).as_uri()
# create a changelist from the files in test/rs/source/{src}/files based on ^that
client = Client(checksum=checksum)
prefix = "http://localhost:8000/rs/source/" + src + "/files"
resourcedir = os.path.join(abs_path, "rs/source", src, "files")
args = [prefix, resourcedir]
client.set_mappings(args)
client.write_change_list(paths=paths, outfile=outfile, ref_sitemap=ref_sitemap)
def sync_audit(map, counter):
"""Run resync audit."""
client = Client()
# ignore fail to continue running, log later
client.ignore_failures = True
client.set_mappings(map)
# init_logging(verbose=True)
src_resource_list = client.find_resource_list()
rlb = ResourceListBuilder(mapper=client.mapper)
dst_resource_list = rlb.from_disk()
# Compare these resource lists respecting any comparison options
(same, updated, deleted,
created) = dst_resource_list.compare(src_resource_list)
result = dict(created=[], updated=[], deleted=[])
for item in created:
record_id = item.uri.rsplit('/', 1)[1]
result['created'].append(record_id)
for item in updated:
record_id = item.uri.rsplit('/', 1)[1]
result['updated'].append(record_id)
for item in deleted:
record_id = item.uri.rsplit('/', 1)[1]
result['deleted'].append(record_id)
update_counter(counter, result)
return dict(same=len(same),
updated=len(updated),
deleted=len(deleted),
created=len(created))
def test1_make_inventory_empty(self):
c = Client()
# No mapping is error
self.assertRaises( TypeError, c.inventory )
# Supply empty mapping
c.set_mappings( [] )
i = c.inventory
self.assertEqual( len(i), 0 )
def test40_write_resource_list_mappings(self):
c = Client()
c.set_mappings( ['http://example.org/','resync/test/testdata/'] )
# with no explicit paths seting the mapping will be used
with capture_stdout() as capturer:
c.write_resource_list()
#sys.stderr.write(capturer.result)
self.assertTrue( re.search(r'<rs:md at="\S+" capability="resourcelist"', capturer.result ) )
self.assertTrue( re.search(r'<url><loc>http://example.org/dir1/file_a</loc>', capturer.result ) )
self.assertTrue( re.search(r'<url><loc>http://example.org/dir1/file_b</loc>', capturer.result ) )
self.assertTrue( re.search(r'<url><loc>http://example.org/dir2/file_x</loc>', capturer.result ) )
def test41_write_resource_list_path(self):
c = Client()
c.set_mappings( ['http://example.org/','resync/test/testdata/'] )
# with an explicit paths setting only the specified paths will be included
with capture_stdout() as capturer:
c.write_resource_list(paths='resync/test/testdata/dir1')
sys.stderr.write(capturer.result)
self.assertTrue( re.search(r'<rs:md at="\S+" capability="resourcelist"', capturer.result ) )
self.assertTrue( re.search(r'<url><loc>http://example.org/dir1/file_a</loc>', capturer.result ) )
self.assertTrue( re.search(r'<url><loc>http://example.org/dir1/file_b</loc>', capturer.result ) )
self.assertFalse( re.search(r'<url><loc>http://example.org/dir2/file_x</loc>', capturer.result ) )
def test10_baseline_or_audit(self):
# FIXME - this is the guts of the client, tough to test, need to work through more cases...
c = Client()
dst = os.path.join(self.tmpdir,'dst_dir1')
c.set_mappings( ['file:tests/testdata/client/dir1',dst] )
# audit with empty dst, should say 3 to create
with LogCapture() as lc:
c.baseline_or_audit(audit_only=True)
self.assertTrue( re.match(r'Status:\s+NOT IN SYNC.*to create=3', lc.records[-2].msg) )
self.assertEqual( lc.records[-1].msg, 'Completed audit' )
# now do the sync
with LogCapture() as lc:
c.baseline_or_audit()
self.assertTrue( re.match(r'Status:\s+SYNCED.*created=3', lc.records[-2].msg) )
self.assertEqual( lc.records[-1].msg, 'Completed baseline sync' )
def test45_write_change_list(self):
c = Client()
ex1 = 'tests/testdata/examples_from_spec/resourcesync_ex_1.xml'
with capture_stdout() as capturer:
c.write_change_list(ref_sitemap=ex1, newref_sitemap=ex1)
self.assertTrue( re.search(r'<rs:md capability="changelist"', capturer.result) )
# compare ex1 with testdata on disk
c.set_mappings( ['http://example.org/','tests/testdata/'] )
with capture_stdout() as capturer:
c.write_change_list(ref_sitemap=ex1, paths='tests/testdata/dir1')
self.assertTrue( re.search(r'<rs:md capability="changelist"', capturer.result) )
self.assertTrue( re.search(r'<url><loc>http://example.com/res1</loc><rs:md change="deleted" /></url>', capturer.result) )
# to file
outfile = os.path.join(self.tmpdir,'cl_out.xml')
c.write_change_list(ref_sitemap=ex1, newref_sitemap=ex1, outfile=outfile)
self.assertTrue( os.path.getsize(outfile)>100 )
def test41_write_resource_list_path(self):
c = Client()
c.set_mappings(['http://example.org/', 'tests/testdata/'])
links = [{'rel': 'uri_c', 'href': 'uri_d'}]
# with an explicit paths setting only the specified paths will be
# included
with capture_stdout() as capturer:
c.write_resource_list(paths='tests/testdata/dir1', links=links)
self.assertTrue(
re.search(r'<rs:md at="\S+" capability="resourcelist"',
capturer.result))
self.assertTrue(
re.search(r'<url><loc>http://example.org/dir1/file_a</loc>',
capturer.result))
self.assertTrue(
re.search(r'<url><loc>http://example.org/dir1/file_b</loc>',
capturer.result))
self.assertFalse(
re.search(r'<url><loc>http://example.org/dir2/file_x</loc>',
capturer.result))
# check link present
self.assertTrue(re.search(r'rel="uri_c"', capturer.result))
self.assertTrue(re.search(r'href="uri_d"', capturer.result))
# Travis CI does not preserve timestamps from github so test here for the file
# size but not the datestamp
# self.assertTrue( re.search(r'<url><loc>http://example.org/dir1/file_a</loc><lastmod>[\w\-:]+</lastmod><rs:md length="20" /></url>', capturer.result ) )
# self.assertTrue( re.search(r'<url><loc>http://example.org/dir1/file_b</loc><lastmod>[\w\-:]+</lastmod><rs:md length="45" /></url>', capturer.result ) )
# to file
outfile = os.path.join(self.tmpdir, 'rl_out.xml')
c.write_resource_list(paths='tests/testdata/dir1', outfile=outfile)
self.assertTrue(os.path.getsize(outfile) > 100)
# dump instead (default file)
c.default_resource_dump = os.path.join(self.tmpdir, 'rl_out_dump_def')
outfile = c.default_resource_dump + '00000.zip'
self.assertFalse(os.path.exists(outfile))
c.write_resource_list(paths='tests/testdata/dir1', dump=True)
self.assertTrue(os.path.getsize(outfile) > 100)
# (specific file)
outbase = os.path.join(self.tmpdir, 'rl_out_dump')
outfile = outbase + '00000.zip'
self.assertFalse(os.path.exists(outfile))
c.write_resource_list(paths='tests/testdata/dir1',
dump=True,
outfile=outbase)
self.assertTrue(os.path.getsize(outfile) > 100)
def test10_baseline_or_audit(self):
# FIXME - this is the guts of the client, tough to test, need to work through more cases...
c = Client()
dst = os.path.join(self.tmpdir, 'dst_dir1')
c.set_mappings(['file:tests/testdata/client/dir1', dst])
# audit with empty dst, should say 3 to create
with LogCapture() as lc:
c.baseline_or_audit(audit_only=True)
self.assertTrue(
re.match(r'Status:\s+NOT IN SYNC.*to create=3',
lc.records[-2].msg))
self.assertEqual(lc.records[-1].msg, 'Completed audit')
# now do the sync
with LogCapture() as lc:
c.baseline_or_audit()
self.assertTrue(
re.match(r'Status:\s+SYNCED.*created=3', lc.records[-2].msg))
self.assertEqual(lc.records[-1].msg, 'Completed baseline sync')
def test41_write_resource_list_path(self):
c = Client()
c.set_mappings(['http://example.org/', 'resync/test/testdata/'])
# with an explicit paths setting only the specified paths will be included
with capture_stdout() as capturer:
c.write_resource_list(paths='resync/test/testdata/dir1')
sys.stderr.write(capturer.result)
self.assertTrue(
re.search(r'<rs:md at="\S+" capability="resourcelist"',
capturer.result))
self.assertTrue(
re.search(r'<url><loc>http://example.org/dir1/file_a</loc>',
capturer.result))
self.assertTrue(
re.search(r'<url><loc>http://example.org/dir1/file_b</loc>',
capturer.result))
self.assertFalse(
re.search(r'<url><loc>http://example.org/dir2/file_x</loc>',
capturer.result))
def test40_write_resource_list_mappings(self):
c = Client()
c.set_mappings(['http://example.org/', 'resync/test/testdata/'])
# with no explicit paths seting the mapping will be used
with capture_stdout() as capturer:
c.write_resource_list()
#sys.stderr.write(capturer.result)
self.assertTrue(
re.search(r'<rs:md at="\S+" capability="resourcelist"',
capturer.result))
self.assertTrue(
re.search(r'<url><loc>http://example.org/dir1/file_a</loc>',
capturer.result))
self.assertTrue(
re.search(r'<url><loc>http://example.org/dir1/file_b</loc>',
capturer.result))
self.assertTrue(
re.search(r'<url><loc>http://example.org/dir2/file_x</loc>',
capturer.result))
def test45_write_change_list(self):
c = Client()
ex1 = 'tests/testdata/examples_from_spec/resourcesync_ex_1.xml'
with capture_stdout() as capturer:
c.write_change_list(ref_sitemap=ex1, newref_sitemap=ex1)
self.assertTrue(
re.search(r'<rs:md capability="changelist"', capturer.result))
# compare ex1 with testdata on disk
c.set_mappings(['http://example.org/', 'tests/testdata/'])
with capture_stdout() as capturer:
c.write_change_list(ref_sitemap=ex1, paths='tests/testdata/dir1')
self.assertTrue(
re.search(r'<rs:md capability="changelist"', capturer.result))
self.assertTrue(
re.search(
r'<url><loc>http://example.com/res1</loc><rs:md change="deleted" /></url>',
capturer.result))
# to file
outfile = os.path.join(self.tmpdir, 'cl_out.xml')
c.write_change_list(ref_sitemap=ex1,
newref_sitemap=ex1,
outfile=outfile)
self.assertTrue(os.path.getsize(outfile) > 100)
def test2_bad_source_uri(self):
c = Client()
self.assertRaises( ClientFatalError, c.baseline_or_audit )
c.set_mappings( ['http://example.org/bbb','/tmp/this_does_not_exist'] )
self.assertRaises( ClientFatalError, c.baseline_or_audit )
def test2_bad_source_uri(self):
c = Client(logger=logging.getLogger())
self.assertRaises( ClientFatalError, c.sync_or_audit )
c.set_mappings( ['http://example.org/bbb','/tmp/this_does_not_exist'] )
self.assertRaises( ClientFatalError, c.sync_or_audit )
def test05_find_resource_list(self):
c = Client()
# Filesystem tests
c.set_mappings(['tests/testdata/find/find1', 'xxx'])
self.assertEqual(c.find_resource_list().up, 'find1')
c.set_mappings(['tests/testdata/find/find2', 'xxx'])
self.assertEqual(c.find_resource_list().up, 'find2')
c.set_mappings(['tests/testdata/find/find3', 'xxx'])
self.assertEqual(c.find_resource_list().up, 'find3')
# Tests requiring a server
with webserver('tests/testdata/find', 'localhost', 9999):
c.set_mappings(['http://localhost:9999/find1', 'xxx'])
self.assertEqual(c.find_resource_list().up, 'find1')
c.set_mappings(['http://localhost:9999/find2', 'xxx'])
self.assertEqual(c.find_resource_list().up, 'find2')
c.set_mappings(['http://localhost:9999/find3', 'xxx'])
self.assertEqual(c.find_resource_list().up, 'find3')
with webserver('tests/testdata/find/find1', 'localhost', 9999):
c.set_mappings(['http://localhost:9999/data', 'xxx'])
self.assertEqual(c.find_resource_list().up, 'find1')
with webserver('tests/testdata/find/find3', 'localhost', 9999):
c.set_mappings(['http://localhost:9999/data/data1', 'xxx'])
self.assertEqual(c.find_resource_list().up, 'find3')
def test02_bad_source_uri(self):
c = Client()
self.assertRaises(ClientFatalError, c.baseline_or_audit)
c.set_mappings(['http://example.org/bbb', '/tmp/this_does_not_exist'])
self.assertRaises(ClientFatalError, c.baseline_or_audit)
def test03_sitemap_uri(self):
c = Client()
c.set_mappings(['http://example.org/c', '/tmp/not_there_at_all'])
self.assertEqual(c.sitemap_uri('abcd1'), 'http://example.org/c/abcd1')
self.assertEqual(c.sitemap_uri('/abcd2'), '/abcd2')
self.assertEqual(c.sitemap_uri('scheme:/abcd3'), 'scheme:/abcd3')
def test2_bad_source_uri(self):
c = Client(logger=logging.getLogger())
self.assertRaises(ClientFatalError, c.sync_or_audit)
c.set_mappings(['http://example.org/bbb', '/tmp/this_does_not_exist'])
self.assertRaises(ClientFatalError, c.sync_or_audit)
def test05_find_resource_list(self):
c = Client()
# Filesystem tests
c.set_mappings(['tests/testdata/find/find1', 'xxx'])
self.assertEqual(c.find_resource_list().up, 'find1')
c.set_mappings(['tests/testdata/find/find2', 'xxx'])
self.assertEqual(c.find_resource_list().up, 'find2')
c.set_mappings(['tests/testdata/find/find3', 'xxx'])
self.assertEqual(c.find_resource_list().up, 'find3')
# Tests requiring a server
with webserver('tests/testdata/find', 'localhost', 9999):
c.set_mappings(['http://localhost:9999/find1', 'xxx'])
self.assertEqual(c.find_resource_list().up, 'find1')
c.set_mappings(['http://localhost:9999/find2', 'xxx'])
self.assertEqual(c.find_resource_list().up, 'find2')
c.set_mappings(['http://localhost:9999/find3', 'xxx'])
self.assertEqual(c.find_resource_list().up, 'find3')
with webserver('tests/testdata/find/find1', 'localhost', 9999):
c.set_mappings(['http://localhost:9999/data', 'xxx'])
self.assertEqual(c.find_resource_list().up, 'find1')
with webserver('tests/testdata/find/find3', 'localhost', 9999):
c.set_mappings(['http://localhost:9999/data/data1', 'xxx'])
self.assertEqual(c.find_resource_list().up, 'find3')
def test03_sitemap_uri(self):
c = Client()
c.set_mappings( ['http://example.org/c','/tmp/not_there_at_all'] )
self.assertEqual( c.sitemap_uri('abcd1'), 'http://example.org/c/abcd1' )
self.assertEqual( c.sitemap_uri('/abcd2'), '/abcd2' )
self.assertEqual( c.sitemap_uri('scheme:/abcd3'), 'scheme:/abcd3' )