Esempio n. 1
0
def __create_resourcelist__(src, checksum=True):
    """
    Create a resourcelist for the source denominated by src.
    :param src: 's1', 's2', 'foo' etc.
    :param checksum: should checksums be added to the xml.
    :return: resync.resource_list.ResourceList
    """
    abs_path = os.path.dirname(os.path.abspath(__name__))
    data = []
    path = os.path.join(abs_path, "rs/source", src, "files")
    for root, directories, filenames in os.walk(path):
        for filename in filenames:
            data.append(os.path.join(root,filename))

    paths = ",".join(data)

    # create a resourcelist from the files in test/rs/source/{src}/files
    client = Client(checksum=checksum)
    prefix = "http://localhost:8000/rs/source/" + src + "/files"
    resourcedir = os.path.join(abs_path, "rs/source", src, "files")
    args = [prefix, resourcedir]

    client.set_mappings(args)
    rl = client.build_resource_list(paths=paths, set_path=True)
    return rl
Esempio n. 2
0
 def test41_write_resource_list_path(self):
     c = Client()
     c.set_mappings( ['http://example.org/','tests/testdata/'] )
     links=[{'rel':'uri_c','href':'uri_d'}]
     # with an explicit paths setting only the specified paths will be included
     with capture_stdout() as capturer:
         c.write_resource_list(paths='tests/testdata/dir1', links=links)
     self.assertTrue( re.search(r'<rs:md at="\S+" capability="resourcelist"', capturer.result ) )
     self.assertTrue( re.search(r'<url><loc>http://example.org/dir1/file_a</loc>', capturer.result ) )
     self.assertTrue( re.search(r'<url><loc>http://example.org/dir1/file_b</loc>', capturer.result ) )
     self.assertFalse( re.search(r'<url><loc>http://example.org/dir2/file_x</loc>', capturer.result ) )
     # check link present
     self.assertTrue( re.search(r'rel="uri_c"', capturer.result ) )
     self.assertTrue( re.search(r'href="uri_d"', capturer.result ) )
     # Travis CI does not preserve timestamps from github so test here for the file
     # size but not the datestamp
     #self.assertTrue( re.search(r'<url><loc>http://example.org/dir1/file_a</loc><lastmod>[\w\-:]+</lastmod><rs:md length="20" /></url>', capturer.result ) )
     #self.assertTrue( re.search(r'<url><loc>http://example.org/dir1/file_b</loc><lastmod>[\w\-:]+</lastmod><rs:md length="45" /></url>', capturer.result ) )
     # to file 
     outfile = os.path.join(self.tmpdir,'rl_out.xml')
     c.write_resource_list(paths='tests/testdata/dir1', outfile=outfile)
     self.assertTrue( os.path.getsize(outfile)>100 )
     # dump instead (default file)
     c.default_resource_dump = os.path.join(self.tmpdir,'rl_out_dump_def')
     outfile = c.default_resource_dump+'00000.zip'
     self.assertFalse( os.path.exists(outfile) )
     c.write_resource_list(paths='tests/testdata/dir1', dump=True)
     self.assertTrue( os.path.getsize(outfile)>100 )
     # (specific file)
     outbase = os.path.join(self.tmpdir,'rl_out_dump')
     outfile = outbase+'00000.zip'
     self.assertFalse( os.path.exists(outfile) )
     c.write_resource_list(paths='tests/testdata/dir1', dump=True, outfile=outbase)
     self.assertTrue( os.path.getsize(outfile)>100 )
Esempio n. 3
0
def __create_resourcelist__(src, checksum=True):
    """
    Create a resourcelist for the source denominated by src.
    :param src: 's1', 's2', 'foo' etc.
    :param checksum: should checksums be added to the xml.
    :return: resync.resource_list.ResourceList
    """
    abs_path = os.path.dirname(os.path.abspath(__name__))
    data = []
    path = os.path.join(abs_path, "rs/source", src, "files")
    for root, directories, filenames in os.walk(path):
        for filename in filenames:
            data.append(os.path.join(root, filename))

    paths = ",".join(data)

    # create a resourcelist from the files in test/rs/source/{src}/files
    client = Client(checksum=checksum)
    prefix = "http://localhost:8000/rs/source/" + src + "/files"
    resourcedir = os.path.join(abs_path, "rs/source", src, "files")
    args = [prefix, resourcedir]

    client.set_mappings(args)
    rl = client.build_resource_list(paths=paths, set_path=True)
    return rl
Esempio n. 4
0
def __create_changelist__(src, checksum=True):
    """
    Create a changelist xml for the source denominated by src.
    :param src: 's1', 's2' etc.
    :param checksum: should checksums be added to the xml.
    :return: None
    """
    abs_path = os.path.dirname(os.path.abspath(__name__))
    data = []
    path = os.path.join(abs_path, "rs/source", src, "files")
    for root, directories, filenames in os.walk(path):
        for filename in filenames:
            data.append(os.path.join(root, filename))

    paths = ",".join(data)

    outfile = os.path.join(abs_path, "rs/source", src, "changelist.xml")
    ref_sitemap = pathlib.Path(
        os.path.join(abs_path, "rs/source", src, "resourcelist.xml")).as_uri()

    # create a changelist from the files in test/rs/source/{src}/files based on ^that
    client = Client(checksum=checksum)
    prefix = "http://localhost:8000/rs/source/" + src + "/files"
    resourcedir = os.path.join(abs_path, "rs/source", src, "files")
    args = [prefix, resourcedir]

    client.set_mappings(args)
    client.write_change_list(paths=paths,
                             outfile=outfile,
                             ref_sitemap=ref_sitemap)
Esempio n. 5
0
def __create_changelist__(src, checksum=True):
    """
    Create a changelist xml for the source denominated by src.
    :param src: 's1', 's2' etc.
    :param checksum: should checksums be added to the xml.
    :return: None
    """
    abs_path = os.path.dirname(os.path.abspath(__name__))
    data = []
    path = os.path.join(abs_path, "rs/source", src, "files")
    for root, directories, filenames in os.walk(path):
        for filename in filenames:
            data.append(os.path.join(root, filename))

    paths = ",".join(data)

    outfile = os.path.join(abs_path, "rs/source", src, "changelist.xml")
    ref_sitemap = pathlib.Path(os.path.join(abs_path, "rs/source", src, "resourcelist.xml")).as_uri()

    # create a changelist from the files in test/rs/source/{src}/files based on ^that
    client = Client(checksum=checksum)
    prefix = "http://localhost:8000/rs/source/" + src + "/files"
    resourcedir = os.path.join(abs_path, "rs/source", src, "files")
    args = [prefix, resourcedir]

    client.set_mappings(args)
    client.write_change_list(paths=paths, outfile=outfile, ref_sitemap=ref_sitemap)
Esempio n. 6
0
def sync_audit(map, counter):
    """Run resync audit."""
    client = Client()
    # ignore fail to continue running, log later
    client.ignore_failures = True
    client.set_mappings(map)
    # init_logging(verbose=True)
    src_resource_list = client.find_resource_list()
    rlb = ResourceListBuilder(mapper=client.mapper)
    dst_resource_list = rlb.from_disk()
    # Compare these resource lists respecting any comparison options
    (same, updated, deleted,
     created) = dst_resource_list.compare(src_resource_list)
    result = dict(created=[], updated=[], deleted=[])
    for item in created:
        record_id = item.uri.rsplit('/', 1)[1]
        result['created'].append(record_id)
    for item in updated:
        record_id = item.uri.rsplit('/', 1)[1]
        result['updated'].append(record_id)
    for item in deleted:
        record_id = item.uri.rsplit('/', 1)[1]
        result['deleted'].append(record_id)
    update_counter(counter, result)
    return dict(same=len(same),
                updated=len(updated),
                deleted=len(deleted),
                created=len(created))
Esempio n. 7
0
 def test1_make_inventory_empty(self):
     c = Client()
     # No mapping is error
     self.assertRaises( TypeError, c.inventory )
     # Supply empty mapping
     c.set_mappings( [] )
     i = c.inventory
     self.assertEqual( len(i), 0 )
Esempio n. 8
0
 def test40_write_resource_list_mappings(self):
     c = Client()
     c.set_mappings( ['http://example.org/','resync/test/testdata/'] )
     # with no explicit paths seting the mapping will be used
     with capture_stdout() as capturer:
         c.write_resource_list()
     #sys.stderr.write(capturer.result)
     self.assertTrue( re.search(r'<rs:md at="\S+" capability="resourcelist"', capturer.result ) )
     self.assertTrue( re.search(r'<url><loc>http://example.org/dir1/file_a</loc>', capturer.result ) )
     self.assertTrue( re.search(r'<url><loc>http://example.org/dir1/file_b</loc>', capturer.result ) )
     self.assertTrue( re.search(r'<url><loc>http://example.org/dir2/file_x</loc>', capturer.result ) )
Esempio n. 9
0
 def test41_write_resource_list_path(self):
     c = Client()
     c.set_mappings( ['http://example.org/','resync/test/testdata/'] )
     # with an explicit paths setting only the specified paths will be included
     with capture_stdout() as capturer:
         c.write_resource_list(paths='resync/test/testdata/dir1')
     sys.stderr.write(capturer.result)
     self.assertTrue( re.search(r'<rs:md at="\S+" capability="resourcelist"', capturer.result ) )
     self.assertTrue( re.search(r'<url><loc>http://example.org/dir1/file_a</loc>', capturer.result ) )
     self.assertTrue( re.search(r'<url><loc>http://example.org/dir1/file_b</loc>', capturer.result ) )
     self.assertFalse( re.search(r'<url><loc>http://example.org/dir2/file_x</loc>', capturer.result ) )
Esempio n. 10
0
 def test10_baseline_or_audit(self):
     # FIXME - this is the guts of the client, tough to test, need to work through more cases...
     c = Client()
     dst = os.path.join(self.tmpdir,'dst_dir1')
     c.set_mappings( ['file:tests/testdata/client/dir1',dst] )
     # audit with empty dst, should say 3 to create
     with LogCapture() as lc:
         c.baseline_or_audit(audit_only=True)
         self.assertTrue( re.match(r'Status:\s+NOT IN SYNC.*to create=3', lc.records[-2].msg) )
         self.assertEqual( lc.records[-1].msg, 'Completed audit' )
     # now do the sync
     with LogCapture() as lc:
         c.baseline_or_audit()
         self.assertTrue( re.match(r'Status:\s+SYNCED.*created=3', lc.records[-2].msg) )
         self.assertEqual( lc.records[-1].msg, 'Completed baseline sync' )
Esempio n. 11
0
 def test45_write_change_list(self):
     c = Client()
     ex1 = 'tests/testdata/examples_from_spec/resourcesync_ex_1.xml'
     with capture_stdout() as capturer:
         c.write_change_list(ref_sitemap=ex1, newref_sitemap=ex1)
     self.assertTrue( re.search(r'<rs:md capability="changelist"', capturer.result) )
     # compare ex1 with testdata on disk
     c.set_mappings( ['http://example.org/','tests/testdata/'] )
     with capture_stdout() as capturer:
         c.write_change_list(ref_sitemap=ex1, paths='tests/testdata/dir1')
     self.assertTrue( re.search(r'<rs:md capability="changelist"', capturer.result) )
     self.assertTrue( re.search(r'<url><loc>http://example.com/res1</loc><rs:md change="deleted" /></url>', capturer.result) )
     # to file 
     outfile = os.path.join(self.tmpdir,'cl_out.xml')
     c.write_change_list(ref_sitemap=ex1, newref_sitemap=ex1, outfile=outfile)
     self.assertTrue( os.path.getsize(outfile)>100 )
Esempio n. 12
0
 def test41_write_resource_list_path(self):
     c = Client()
     c.set_mappings(['http://example.org/', 'tests/testdata/'])
     links = [{'rel': 'uri_c', 'href': 'uri_d'}]
     # with an explicit paths setting only the specified paths will be
     # included
     with capture_stdout() as capturer:
         c.write_resource_list(paths='tests/testdata/dir1', links=links)
     self.assertTrue(
         re.search(r'<rs:md at="\S+" capability="resourcelist"',
                   capturer.result))
     self.assertTrue(
         re.search(r'<url><loc>http://example.org/dir1/file_a</loc>',
                   capturer.result))
     self.assertTrue(
         re.search(r'<url><loc>http://example.org/dir1/file_b</loc>',
                   capturer.result))
     self.assertFalse(
         re.search(r'<url><loc>http://example.org/dir2/file_x</loc>',
                   capturer.result))
     # check link present
     self.assertTrue(re.search(r'rel="uri_c"', capturer.result))
     self.assertTrue(re.search(r'href="uri_d"', capturer.result))
     # Travis CI does not preserve timestamps from github so test here for the file
     # size but not the datestamp
     # self.assertTrue( re.search(r'<url><loc>http://example.org/dir1/file_a</loc><lastmod>[\w\-:]+</lastmod><rs:md length="20" /></url>', capturer.result ) )
     # self.assertTrue( re.search(r'<url><loc>http://example.org/dir1/file_b</loc><lastmod>[\w\-:]+</lastmod><rs:md length="45" /></url>', capturer.result ) )
     # to file
     outfile = os.path.join(self.tmpdir, 'rl_out.xml')
     c.write_resource_list(paths='tests/testdata/dir1', outfile=outfile)
     self.assertTrue(os.path.getsize(outfile) > 100)
     # dump instead (default file)
     c.default_resource_dump = os.path.join(self.tmpdir, 'rl_out_dump_def')
     outfile = c.default_resource_dump + '00000.zip'
     self.assertFalse(os.path.exists(outfile))
     c.write_resource_list(paths='tests/testdata/dir1', dump=True)
     self.assertTrue(os.path.getsize(outfile) > 100)
     # (specific file)
     outbase = os.path.join(self.tmpdir, 'rl_out_dump')
     outfile = outbase + '00000.zip'
     self.assertFalse(os.path.exists(outfile))
     c.write_resource_list(paths='tests/testdata/dir1',
                           dump=True,
                           outfile=outbase)
     self.assertTrue(os.path.getsize(outfile) > 100)
Esempio n. 13
0
 def test10_baseline_or_audit(self):
     # FIXME - this is the guts of the client, tough to test, need to work through more cases...
     c = Client()
     dst = os.path.join(self.tmpdir, 'dst_dir1')
     c.set_mappings(['file:tests/testdata/client/dir1', dst])
     # audit with empty dst, should say 3 to create
     with LogCapture() as lc:
         c.baseline_or_audit(audit_only=True)
         self.assertTrue(
             re.match(r'Status:\s+NOT IN SYNC.*to create=3',
                      lc.records[-2].msg))
         self.assertEqual(lc.records[-1].msg, 'Completed audit')
     # now do the sync
     with LogCapture() as lc:
         c.baseline_or_audit()
         self.assertTrue(
             re.match(r'Status:\s+SYNCED.*created=3', lc.records[-2].msg))
         self.assertEqual(lc.records[-1].msg, 'Completed baseline sync')
Esempio n. 14
0
 def test41_write_resource_list_path(self):
     c = Client()
     c.set_mappings(['http://example.org/', 'resync/test/testdata/'])
     # with an explicit paths setting only the specified paths will be included
     with capture_stdout() as capturer:
         c.write_resource_list(paths='resync/test/testdata/dir1')
     sys.stderr.write(capturer.result)
     self.assertTrue(
         re.search(r'<rs:md at="\S+" capability="resourcelist"',
                   capturer.result))
     self.assertTrue(
         re.search(r'<url><loc>http://example.org/dir1/file_a</loc>',
                   capturer.result))
     self.assertTrue(
         re.search(r'<url><loc>http://example.org/dir1/file_b</loc>',
                   capturer.result))
     self.assertFalse(
         re.search(r'<url><loc>http://example.org/dir2/file_x</loc>',
                   capturer.result))
Esempio n. 15
0
 def test40_write_resource_list_mappings(self):
     c = Client()
     c.set_mappings(['http://example.org/', 'resync/test/testdata/'])
     # with no explicit paths seting the mapping will be used
     with capture_stdout() as capturer:
         c.write_resource_list()
     #sys.stderr.write(capturer.result)
     self.assertTrue(
         re.search(r'<rs:md at="\S+" capability="resourcelist"',
                   capturer.result))
     self.assertTrue(
         re.search(r'<url><loc>http://example.org/dir1/file_a</loc>',
                   capturer.result))
     self.assertTrue(
         re.search(r'<url><loc>http://example.org/dir1/file_b</loc>',
                   capturer.result))
     self.assertTrue(
         re.search(r'<url><loc>http://example.org/dir2/file_x</loc>',
                   capturer.result))
Esempio n. 16
0
 def test45_write_change_list(self):
     c = Client()
     ex1 = 'tests/testdata/examples_from_spec/resourcesync_ex_1.xml'
     with capture_stdout() as capturer:
         c.write_change_list(ref_sitemap=ex1, newref_sitemap=ex1)
     self.assertTrue(
         re.search(r'<rs:md capability="changelist"', capturer.result))
     # compare ex1 with testdata on disk
     c.set_mappings(['http://example.org/', 'tests/testdata/'])
     with capture_stdout() as capturer:
         c.write_change_list(ref_sitemap=ex1, paths='tests/testdata/dir1')
     self.assertTrue(
         re.search(r'<rs:md capability="changelist"', capturer.result))
     self.assertTrue(
         re.search(
             r'<url><loc>http://example.com/res1</loc><rs:md change="deleted" /></url>',
             capturer.result))
     # to file
     outfile = os.path.join(self.tmpdir, 'cl_out.xml')
     c.write_change_list(ref_sitemap=ex1,
                         newref_sitemap=ex1,
                         outfile=outfile)
     self.assertTrue(os.path.getsize(outfile) > 100)
Esempio n. 17
0
 def test2_bad_source_uri(self):
     c = Client()
     self.assertRaises( ClientFatalError, c.baseline_or_audit )
     c.set_mappings( ['http://example.org/bbb','/tmp/this_does_not_exist'] )
     self.assertRaises( ClientFatalError, c.baseline_or_audit )
Esempio n. 18
0
 def test2_bad_source_uri(self):
     c = Client(logger=logging.getLogger())
     self.assertRaises( ClientFatalError, c.sync_or_audit )
     c.set_mappings( ['http://example.org/bbb','/tmp/this_does_not_exist'] )
     self.assertRaises( ClientFatalError, c.sync_or_audit )
Esempio n. 19
0
 def test05_find_resource_list(self):
     c = Client()
     # Filesystem tests
     c.set_mappings(['tests/testdata/find/find1', 'xxx'])
     self.assertEqual(c.find_resource_list().up, 'find1')
     c.set_mappings(['tests/testdata/find/find2', 'xxx'])
     self.assertEqual(c.find_resource_list().up, 'find2')
     c.set_mappings(['tests/testdata/find/find3', 'xxx'])
     self.assertEqual(c.find_resource_list().up, 'find3')
     # Tests requiring a server
     with webserver('tests/testdata/find', 'localhost', 9999):
         c.set_mappings(['http://localhost:9999/find1', 'xxx'])
         self.assertEqual(c.find_resource_list().up, 'find1')
         c.set_mappings(['http://localhost:9999/find2', 'xxx'])
         self.assertEqual(c.find_resource_list().up, 'find2')
         c.set_mappings(['http://localhost:9999/find3', 'xxx'])
         self.assertEqual(c.find_resource_list().up, 'find3')
     with webserver('tests/testdata/find/find1', 'localhost', 9999):
         c.set_mappings(['http://localhost:9999/data', 'xxx'])
         self.assertEqual(c.find_resource_list().up, 'find1')
     with webserver('tests/testdata/find/find3', 'localhost', 9999):
         c.set_mappings(['http://localhost:9999/data/data1', 'xxx'])
         self.assertEqual(c.find_resource_list().up, 'find3')
Esempio n. 20
0
 def test02_bad_source_uri(self):
     c = Client()
     self.assertRaises(ClientFatalError, c.baseline_or_audit)
     c.set_mappings(['http://example.org/bbb', '/tmp/this_does_not_exist'])
     self.assertRaises(ClientFatalError, c.baseline_or_audit)
Esempio n. 21
0
 def test03_sitemap_uri(self):
     c = Client()
     c.set_mappings(['http://example.org/c', '/tmp/not_there_at_all'])
     self.assertEqual(c.sitemap_uri('abcd1'), 'http://example.org/c/abcd1')
     self.assertEqual(c.sitemap_uri('/abcd2'), '/abcd2')
     self.assertEqual(c.sitemap_uri('scheme:/abcd3'), 'scheme:/abcd3')
Esempio n. 22
0
 def test2_bad_source_uri(self):
     c = Client(logger=logging.getLogger())
     self.assertRaises(ClientFatalError, c.sync_or_audit)
     c.set_mappings(['http://example.org/bbb', '/tmp/this_does_not_exist'])
     self.assertRaises(ClientFatalError, c.sync_or_audit)
Esempio n. 23
0
 def test05_find_resource_list(self):
     c = Client()
     # Filesystem tests
     c.set_mappings(['tests/testdata/find/find1', 'xxx'])
     self.assertEqual(c.find_resource_list().up, 'find1')
     c.set_mappings(['tests/testdata/find/find2', 'xxx'])
     self.assertEqual(c.find_resource_list().up, 'find2')
     c.set_mappings(['tests/testdata/find/find3', 'xxx'])
     self.assertEqual(c.find_resource_list().up, 'find3')
     # Tests requiring a server
     with webserver('tests/testdata/find', 'localhost', 9999):
         c.set_mappings(['http://localhost:9999/find1', 'xxx'])
         self.assertEqual(c.find_resource_list().up, 'find1')
         c.set_mappings(['http://localhost:9999/find2', 'xxx'])
         self.assertEqual(c.find_resource_list().up, 'find2')
         c.set_mappings(['http://localhost:9999/find3', 'xxx'])
         self.assertEqual(c.find_resource_list().up, 'find3')
     with webserver('tests/testdata/find/find1', 'localhost', 9999):
         c.set_mappings(['http://localhost:9999/data', 'xxx'])
         self.assertEqual(c.find_resource_list().up, 'find1')
     with webserver('tests/testdata/find/find3', 'localhost', 9999):
         c.set_mappings(['http://localhost:9999/data/data1', 'xxx'])
         self.assertEqual(c.find_resource_list().up, 'find3')
Esempio n. 24
0
 def test03_sitemap_uri(self):
     c = Client()
     c.set_mappings( ['http://example.org/c','/tmp/not_there_at_all'] )
     self.assertEqual( c.sitemap_uri('abcd1'), 'http://example.org/c/abcd1' )
     self.assertEqual( c.sitemap_uri('/abcd2'), '/abcd2' )
     self.assertEqual( c.sitemap_uri('scheme:/abcd3'), 'scheme:/abcd3' )