def test_get_pathlist_with_subpaths(self):
""" Tests the functionality of get_pathlist with an exclude rule and subpaths"""
regex = re.compile('/\.snapshots(/.*|$)')
res = [('/tree', 0), ('/tree/c1', 0), ('/tree/b1', 0), ('/tree/a1', 0), ('/tree/b1/bb2', 0),
('/tree/b1/ba2', 0), ('/tree/a1/ac2', 0), ('/tree/a1/ab2', 0), ('/tree/a1/aa2', 0),
('/tree/a1/ab2/aa3', 0), ('/tree/a1/ab2/aa3/sub1', 0), ('/tree/a1/ab2/aa3/sub2', 0),
('/tree/a1/ab2/aa3/sub2/sub21', 0), ('/tree/a1/ab2/aa3/sub2/sub21/sub211', 1), ]
subpath1 = '3_a1/ab2/aa3'
subpath2 = '3_a1/ab2/aa3/sub2'
self.assertRaises(Exception, dw.get_pathlist, self.basedir, 3, exclude_re=regex, exclude_usr=None, rsubpaths=['3_/a1/ab2/aa3'])
self.assertRaises(Exception, dw.get_pathlist, self.basedir, 2, exclude_re=regex, exclude_usr=None, rsubpaths=['3_a1/ab2/aa3'])
self.assertRaises(Exception, dw.get_pathlist, self.basedir, 3, exclude_re=regex, exclude_usr=None, rsubpaths=['1_a1'])
genlist = [(re.sub('(/var)?/tmp/[^/]+', '/tree', gen), rec) for (gen, rec) in dw.get_pathlist(self.basedir, 3, exclude_re=regex, rsubpaths=[subpath1])]
self.assertEqual(sorted(genlist), sorted(res))
genlist = [(re.sub('(/var)?/tmp/[^/]+', '/tree', gen), rec)
for (gen, rec) in dw.get_pathlist(self.basedir, 3, exclude_re=regex, rsubpaths=[subpath1, subpath2])]
res.remove(('/tree/a1/ab2/aa3/sub2/sub21/sub211', 1))
res.extend([('/tree/a1/ab2/aa3/sub2/sub21/sub211', 0), ('/tree/a1/ab2/aa3/sub2/sub21/sub211/sub2112', 1)])
self.assertEqual(sorted(genlist), sorted(res))
subpath2 = '4_a1/ab2/aa3/sub2'
genlist = [(re.sub('(/var)?/tmp/[^/]+', '/tree', gen), rec)
for (gen, rec) in dw.get_pathlist(self.basedir, 3, exclude_re=regex, rsubpaths=[subpath1, subpath2])]
res.remove(('/tree/a1/ab2/aa3/sub2/sub21/sub211/sub2112', 1))
res.append(('/tree/a1/ab2/aa3/sub2/sub21/sub211/sub2112', 0))
self.assertEqual(sorted(genlist), sorted(res))
subpath1 = '3_a1/ab2'
subpath2 = '3_a1/ab2/aa3'
self.assertRaises(Exception, dw.get_pathlist, self.basedir, 3, exclude_re=regex, exclude_usr=None, rsubpaths=[subpath2, subpath1])
def test_get_pathlist(self):
""" Tests the functionality of get_pathlist with an exclude rule """
regex = re.compile('/\.snapshots(/.*|$)')
res = [('/tree', 0), ('/tree/c1', 0), ('/tree/b1', 0), ('/tree/a1', 0), ('/tree/b1/bb2', 0),
('/tree/b1/ba2', 0), ('/tree/a1/ac2', 0), ('/tree/a1/ab2', 0), ('/tree/a1/aa2', 0),
('/tree/a1/ab2/aa3', 1)]
genlist = [(re.sub('/tmp/[^/]+', '/tree', gen), rec) for (gen, rec) in dw.get_pathlist(self.basedir, 3, exclude_re=regex, exclude_usr=None)]
self.assertListEqual(genlist , res)
def test_get_pathlist(self):
""" Tests the functionality of get_pathlist with an exclude rule """
regex = re.compile('/\.snapshots(/.*|$)')
res = [('/tree', 0), ('/tree/c1', 0), ('/tree/b1', 0), ('/tree/a1', 0),
('/tree/b1/bb2', 0), ('/tree/b1/ba2', 0), ('/tree/a1/ac2', 0),
('/tree/a1/ab2', 0), ('/tree/a1/aa2', 0),
('/tree/a1/ab2/aa3', 1)]
genlist = [(re.sub('(/var)?/tmp/[^/]+', '/tree', gen), rec)
for (gen, rec) in dw.get_pathlist(
self.basedir, 3, exclude_re=regex, exclude_usr=None)]
self.assertEqual(sorted(genlist), sorted(res))
def build_pathqueue(self):
""" Build a queue of paths that needs to be rsynced """
self.log.debug('removing old queue and building new queue')
if self.exists(self.path_queue.path):
self.delete(self.path_queue.path, recursive=True)
if self.netcat:
paths = [str(i) for i in range(self.NC_RANGE)]
time.sleep(self.SLEEPTIME)
else:
tuplpaths = get_pathlist(self.rsyncpath, self.rsyncdepth, exclude_re=self.excludere,
exclude_usr='******') # Don't exclude user files
paths = encode_paths(tuplpaths)
self.paths_total = len(paths)
for path in paths:
self.path_queue.put(path) # Put_all can issue a zookeeper connection error with big lists
self.log.debug('pathqueue building finished')
return self.paths_total
def build_pathqueue(self):
""" Build a queue of paths that needs to be rsynced """
self.log.info('removing old queue and building new queue')
if self.exists(self.path_queue.path):
self.delete(self.path_queue.path, recursive=True)
if self.netcat:
paths = [str(i) for i in range(self.NC_RANGE)]
time.sleep(self.SLEEPTIME)
else:
tuplpaths = get_pathlist(
self.rsyncpath,
self.rsyncdepth,
exclude_re=self.excludere,
# By default don't exclude user files
exclude_usr=self.excl_usr,
rsubpaths=self.rsubpaths)
paths = encode_paths(tuplpaths)
self.paths_total = len(paths)
for path in paths:
self.path_queue.put(path.encode(
)) # Put_all can issue a zookeeper connection error with big lists
self.log.info('pathqueue building finished')
return self.paths_total
def test_get_pathlist_with_subpaths(self):
""" Tests the functionality of get_pathlist with an exclude rule and subpaths"""
regex = re.compile('/\.snapshots(/.*|$)')
res = [
('/tree', 0),
('/tree/c1', 0),
('/tree/b1', 0),
('/tree/a1', 0),
('/tree/b1/bb2', 0),
('/tree/b1/ba2', 0),
('/tree/a1/ac2', 0),
('/tree/a1/ab2', 0),
('/tree/a1/aa2', 0),
('/tree/a1/ab2/aa3', 0),
('/tree/a1/ab2/aa3/sub1', 0),
('/tree/a1/ab2/aa3/sub2', 0),
('/tree/a1/ab2/aa3/sub2/sub21', 0),
('/tree/a1/ab2/aa3/sub2/sub21/sub211', 1),
]
subpath1 = '3_a1/ab2/aa3'
subpath2 = '3_a1/ab2/aa3/sub2'
self.assertRaises(Exception,
dw.get_pathlist,
self.basedir,
3,
exclude_re=regex,
exclude_usr=None,
rsubpaths=['3_/a1/ab2/aa3'])
self.assertRaises(Exception,
dw.get_pathlist,
self.basedir,
2,
exclude_re=regex,
exclude_usr=None,
rsubpaths=['3_a1/ab2/aa3'])
self.assertRaises(Exception,
dw.get_pathlist,
self.basedir,
3,
exclude_re=regex,
exclude_usr=None,
rsubpaths=['1_a1'])
genlist = [(re.sub('(/var)?/tmp/[^/]+', '/tree', gen), rec)
for (gen, rec) in dw.get_pathlist(
self.basedir, 3, exclude_re=regex, rsubpaths=[subpath1])
]
self.assertEqual(sorted(genlist), sorted(res))
genlist = [(re.sub('(/var)?/tmp/[^/]+', '/tree', gen), rec) for (
gen, rec
) in dw.get_pathlist(
self.basedir, 3, exclude_re=regex, rsubpaths=[subpath1, subpath2])]
res.remove(('/tree/a1/ab2/aa3/sub2/sub21/sub211', 1))
res.extend([('/tree/a1/ab2/aa3/sub2/sub21/sub211', 0),
('/tree/a1/ab2/aa3/sub2/sub21/sub211/sub2112', 1)])
self.assertEqual(sorted(genlist), sorted(res))
subpath2 = '4_a1/ab2/aa3/sub2'
genlist = [(re.sub('(/var)?/tmp/[^/]+', '/tree', gen), rec) for (
gen, rec
) in dw.get_pathlist(
self.basedir, 3, exclude_re=regex, rsubpaths=[subpath1, subpath2])]
res.remove(('/tree/a1/ab2/aa3/sub2/sub21/sub211/sub2112', 1))
res.append(('/tree/a1/ab2/aa3/sub2/sub21/sub211/sub2112', 0))
self.assertEqual(sorted(genlist), sorted(res))
subpath1 = '3_a1/ab2'
subpath2 = '3_a1/ab2/aa3'
self.assertRaises(Exception,
dw.get_pathlist,
self.basedir,
3,
exclude_re=regex,
exclude_usr=None,
rsubpaths=[subpath2, subpath1])
