def test_get_pathlist_with_subpaths(self): """ Tests the functionality of get_pathlist with an exclude rule and subpaths""" regex = re.compile('/\.snapshots(/.*|$)') res = [('/tree', 0), ('/tree/c1', 0), ('/tree/b1', 0), ('/tree/a1', 0), ('/tree/b1/bb2', 0), ('/tree/b1/ba2', 0), ('/tree/a1/ac2', 0), ('/tree/a1/ab2', 0), ('/tree/a1/aa2', 0), ('/tree/a1/ab2/aa3', 0), ('/tree/a1/ab2/aa3/sub1', 0), ('/tree/a1/ab2/aa3/sub2', 0), ('/tree/a1/ab2/aa3/sub2/sub21', 0), ('/tree/a1/ab2/aa3/sub2/sub21/sub211', 1), ] subpath1 = '3_a1/ab2/aa3' subpath2 = '3_a1/ab2/aa3/sub2' self.assertRaises(Exception, dw.get_pathlist, self.basedir, 3, exclude_re=regex, exclude_usr=None, rsubpaths=['3_/a1/ab2/aa3']) self.assertRaises(Exception, dw.get_pathlist, self.basedir, 2, exclude_re=regex, exclude_usr=None, rsubpaths=['3_a1/ab2/aa3']) self.assertRaises(Exception, dw.get_pathlist, self.basedir, 3, exclude_re=regex, exclude_usr=None, rsubpaths=['1_a1']) genlist = [(re.sub('(/var)?/tmp/[^/]+', '/tree', gen), rec) for (gen, rec) in dw.get_pathlist(self.basedir, 3, exclude_re=regex, rsubpaths=[subpath1])] self.assertEqual(sorted(genlist), sorted(res)) genlist = [(re.sub('(/var)?/tmp/[^/]+', '/tree', gen), rec) for (gen, rec) in dw.get_pathlist(self.basedir, 3, exclude_re=regex, rsubpaths=[subpath1, subpath2])] res.remove(('/tree/a1/ab2/aa3/sub2/sub21/sub211', 1)) res.extend([('/tree/a1/ab2/aa3/sub2/sub21/sub211', 0), ('/tree/a1/ab2/aa3/sub2/sub21/sub211/sub2112', 1)]) self.assertEqual(sorted(genlist), sorted(res)) subpath2 = '4_a1/ab2/aa3/sub2' genlist = [(re.sub('(/var)?/tmp/[^/]+', '/tree', gen), rec) for (gen, rec) in dw.get_pathlist(self.basedir, 3, exclude_re=regex, rsubpaths=[subpath1, subpath2])] res.remove(('/tree/a1/ab2/aa3/sub2/sub21/sub211/sub2112', 1)) res.append(('/tree/a1/ab2/aa3/sub2/sub21/sub211/sub2112', 0)) self.assertEqual(sorted(genlist), sorted(res)) subpath1 = '3_a1/ab2' subpath2 = '3_a1/ab2/aa3' self.assertRaises(Exception, dw.get_pathlist, self.basedir, 3, exclude_re=regex, exclude_usr=None, rsubpaths=[subpath2, subpath1])
def test_get_pathlist(self): """ Tests the functionality of get_pathlist with an exclude rule """ regex = re.compile('/\.snapshots(/.*|$)') res = [('/tree', 0), ('/tree/c1', 0), ('/tree/b1', 0), ('/tree/a1', 0), ('/tree/b1/bb2', 0), ('/tree/b1/ba2', 0), ('/tree/a1/ac2', 0), ('/tree/a1/ab2', 0), ('/tree/a1/aa2', 0), ('/tree/a1/ab2/aa3', 1)] genlist = [(re.sub('/tmp/[^/]+', '/tree', gen), rec) for (gen, rec) in dw.get_pathlist(self.basedir, 3, exclude_re=regex, exclude_usr=None)] self.assertListEqual(genlist , res)
def test_get_pathlist(self): """ Tests the functionality of get_pathlist with an exclude rule """ regex = re.compile('/\.snapshots(/.*|$)') res = [('/tree', 0), ('/tree/c1', 0), ('/tree/b1', 0), ('/tree/a1', 0), ('/tree/b1/bb2', 0), ('/tree/b1/ba2', 0), ('/tree/a1/ac2', 0), ('/tree/a1/ab2', 0), ('/tree/a1/aa2', 0), ('/tree/a1/ab2/aa3', 1)] genlist = [(re.sub('(/var)?/tmp/[^/]+', '/tree', gen), rec) for (gen, rec) in dw.get_pathlist( self.basedir, 3, exclude_re=regex, exclude_usr=None)] self.assertEqual(sorted(genlist), sorted(res))
def build_pathqueue(self): """ Build a queue of paths that needs to be rsynced """ self.log.debug('removing old queue and building new queue') if self.exists(self.path_queue.path): self.delete(self.path_queue.path, recursive=True) if self.netcat: paths = [str(i) for i in range(self.NC_RANGE)] time.sleep(self.SLEEPTIME) else: tuplpaths = get_pathlist(self.rsyncpath, self.rsyncdepth, exclude_re=self.excludere, exclude_usr='******') # Don't exclude user files paths = encode_paths(tuplpaths) self.paths_total = len(paths) for path in paths: self.path_queue.put(path) # Put_all can issue a zookeeper connection error with big lists self.log.debug('pathqueue building finished') return self.paths_total
def build_pathqueue(self): """ Build a queue of paths that needs to be rsynced """ self.log.info('removing old queue and building new queue') if self.exists(self.path_queue.path): self.delete(self.path_queue.path, recursive=True) if self.netcat: paths = [str(i) for i in range(self.NC_RANGE)] time.sleep(self.SLEEPTIME) else: tuplpaths = get_pathlist( self.rsyncpath, self.rsyncdepth, exclude_re=self.excludere, # By default don't exclude user files exclude_usr=self.excl_usr, rsubpaths=self.rsubpaths) paths = encode_paths(tuplpaths) self.paths_total = len(paths) for path in paths: self.path_queue.put(path.encode( )) # Put_all can issue a zookeeper connection error with big lists self.log.info('pathqueue building finished') return self.paths_total
def test_get_pathlist_with_subpaths(self): """ Tests the functionality of get_pathlist with an exclude rule and subpaths""" regex = re.compile('/\.snapshots(/.*|$)') res = [ ('/tree', 0), ('/tree/c1', 0), ('/tree/b1', 0), ('/tree/a1', 0), ('/tree/b1/bb2', 0), ('/tree/b1/ba2', 0), ('/tree/a1/ac2', 0), ('/tree/a1/ab2', 0), ('/tree/a1/aa2', 0), ('/tree/a1/ab2/aa3', 0), ('/tree/a1/ab2/aa3/sub1', 0), ('/tree/a1/ab2/aa3/sub2', 0), ('/tree/a1/ab2/aa3/sub2/sub21', 0), ('/tree/a1/ab2/aa3/sub2/sub21/sub211', 1), ] subpath1 = '3_a1/ab2/aa3' subpath2 = '3_a1/ab2/aa3/sub2' self.assertRaises(Exception, dw.get_pathlist, self.basedir, 3, exclude_re=regex, exclude_usr=None, rsubpaths=['3_/a1/ab2/aa3']) self.assertRaises(Exception, dw.get_pathlist, self.basedir, 2, exclude_re=regex, exclude_usr=None, rsubpaths=['3_a1/ab2/aa3']) self.assertRaises(Exception, dw.get_pathlist, self.basedir, 3, exclude_re=regex, exclude_usr=None, rsubpaths=['1_a1']) genlist = [(re.sub('(/var)?/tmp/[^/]+', '/tree', gen), rec) for (gen, rec) in dw.get_pathlist( self.basedir, 3, exclude_re=regex, rsubpaths=[subpath1]) ] self.assertEqual(sorted(genlist), sorted(res)) genlist = [(re.sub('(/var)?/tmp/[^/]+', '/tree', gen), rec) for ( gen, rec ) in dw.get_pathlist( self.basedir, 3, exclude_re=regex, rsubpaths=[subpath1, subpath2])] res.remove(('/tree/a1/ab2/aa3/sub2/sub21/sub211', 1)) res.extend([('/tree/a1/ab2/aa3/sub2/sub21/sub211', 0), ('/tree/a1/ab2/aa3/sub2/sub21/sub211/sub2112', 1)]) self.assertEqual(sorted(genlist), sorted(res)) subpath2 = '4_a1/ab2/aa3/sub2' genlist = [(re.sub('(/var)?/tmp/[^/]+', '/tree', gen), rec) for ( gen, rec ) in dw.get_pathlist( self.basedir, 3, exclude_re=regex, rsubpaths=[subpath1, subpath2])] res.remove(('/tree/a1/ab2/aa3/sub2/sub21/sub211/sub2112', 1)) res.append(('/tree/a1/ab2/aa3/sub2/sub21/sub211/sub2112', 0)) self.assertEqual(sorted(genlist), sorted(res)) subpath1 = '3_a1/ab2' subpath2 = '3_a1/ab2/aa3' self.assertRaises(Exception, dw.get_pathlist, self.basedir, 3, exclude_re=regex, exclude_usr=None, rsubpaths=[subpath2, subpath1])