Beispiel #1
0
    def test_get_pathlist_with_subpaths(self):
        """ Tests the functionality of get_pathlist with an exclude rule and subpaths"""
        regex = re.compile('/\.snapshots(/.*|$)')
        res = [('/tree', 0), ('/tree/c1', 0), ('/tree/b1', 0), ('/tree/a1', 0), ('/tree/b1/bb2', 0),
               ('/tree/b1/ba2', 0), ('/tree/a1/ac2', 0), ('/tree/a1/ab2', 0), ('/tree/a1/aa2', 0),
               ('/tree/a1/ab2/aa3', 0), ('/tree/a1/ab2/aa3/sub1', 0), ('/tree/a1/ab2/aa3/sub2', 0),
               ('/tree/a1/ab2/aa3/sub2/sub21', 0), ('/tree/a1/ab2/aa3/sub2/sub21/sub211', 1), ]

        subpath1 = '3_a1/ab2/aa3'
        subpath2 = '3_a1/ab2/aa3/sub2'
        self.assertRaises(Exception, dw.get_pathlist, self.basedir, 3, exclude_re=regex, exclude_usr=None, rsubpaths=['3_/a1/ab2/aa3'])
        self.assertRaises(Exception, dw.get_pathlist, self.basedir, 2, exclude_re=regex, exclude_usr=None, rsubpaths=['3_a1/ab2/aa3'])
        self.assertRaises(Exception, dw.get_pathlist, self.basedir, 3, exclude_re=regex, exclude_usr=None, rsubpaths=['1_a1'])

        genlist = [(re.sub('(/var)?/tmp/[^/]+', '/tree', gen), rec) for (gen, rec) in dw.get_pathlist(self.basedir, 3, exclude_re=regex, rsubpaths=[subpath1])]
        self.assertEqual(sorted(genlist), sorted(res))

        genlist = [(re.sub('(/var)?/tmp/[^/]+', '/tree', gen), rec)
                   for (gen, rec) in dw.get_pathlist(self.basedir, 3, exclude_re=regex, rsubpaths=[subpath1, subpath2])]
        res.remove(('/tree/a1/ab2/aa3/sub2/sub21/sub211', 1))
        res.extend([('/tree/a1/ab2/aa3/sub2/sub21/sub211', 0), ('/tree/a1/ab2/aa3/sub2/sub21/sub211/sub2112', 1)])
        self.assertEqual(sorted(genlist), sorted(res))

        subpath2 = '4_a1/ab2/aa3/sub2'
        genlist = [(re.sub('(/var)?/tmp/[^/]+', '/tree', gen), rec)
                   for (gen, rec) in dw.get_pathlist(self.basedir, 3, exclude_re=regex, rsubpaths=[subpath1, subpath2])]
        res.remove(('/tree/a1/ab2/aa3/sub2/sub21/sub211/sub2112', 1))
        res.append(('/tree/a1/ab2/aa3/sub2/sub21/sub211/sub2112', 0))
        self.assertEqual(sorted(genlist), sorted(res))

        subpath1 = '3_a1/ab2'
        subpath2 = '3_a1/ab2/aa3'
        self.assertRaises(Exception, dw.get_pathlist, self.basedir, 3, exclude_re=regex, exclude_usr=None, rsubpaths=[subpath2, subpath1])
Beispiel #2
0
 def test_get_pathlist(self):
     """ Tests the functionality of get_pathlist with an exclude rule """
     regex = re.compile('/\.snapshots(/.*|$)')
     res = [('/tree', 0), ('/tree/c1', 0), ('/tree/b1', 0), ('/tree/a1', 0), ('/tree/b1/bb2', 0),
            ('/tree/b1/ba2', 0), ('/tree/a1/ac2', 0), ('/tree/a1/ab2', 0), ('/tree/a1/aa2', 0),
            ('/tree/a1/ab2/aa3', 1)]
     genlist = [(re.sub('/tmp/[^/]+', '/tree', gen), rec) for (gen, rec) in dw.get_pathlist(self.basedir, 3, exclude_re=regex, exclude_usr=None)]
     self.assertListEqual(genlist , res)
Beispiel #3
0
 def test_get_pathlist(self):
     """ Tests the functionality of get_pathlist with an exclude rule """
     regex = re.compile('/\.snapshots(/.*|$)')
     res = [('/tree', 0), ('/tree/c1', 0), ('/tree/b1', 0), ('/tree/a1', 0),
            ('/tree/b1/bb2', 0), ('/tree/b1/ba2', 0), ('/tree/a1/ac2', 0),
            ('/tree/a1/ab2', 0), ('/tree/a1/aa2', 0),
            ('/tree/a1/ab2/aa3', 1)]
     genlist = [(re.sub('(/var)?/tmp/[^/]+', '/tree', gen), rec)
                for (gen, rec) in dw.get_pathlist(
                    self.basedir, 3, exclude_re=regex, exclude_usr=None)]
     self.assertEqual(sorted(genlist), sorted(res))
Beispiel #4
0
 def build_pathqueue(self):
     """ Build a queue of paths that needs to be rsynced """
     self.log.debug('removing old queue and building new queue')
     if self.exists(self.path_queue.path):
         self.delete(self.path_queue.path, recursive=True)
     if self.netcat:
         paths = [str(i) for i in range(self.NC_RANGE)]
         time.sleep(self.SLEEPTIME)
     else:
         tuplpaths = get_pathlist(self.rsyncpath, self.rsyncdepth, exclude_re=self.excludere,
                                  exclude_usr='******')  # Don't exclude user files
         paths = encode_paths(tuplpaths)
     self.paths_total = len(paths)
     for path in paths:
         self.path_queue.put(path)  # Put_all can issue a zookeeper connection error with big lists
     self.log.debug('pathqueue building finished')
     return self.paths_total
Beispiel #5
0
 def build_pathqueue(self):
     """ Build a queue of paths that needs to be rsynced """
     self.log.info('removing old queue and building new queue')
     if self.exists(self.path_queue.path):
         self.delete(self.path_queue.path, recursive=True)
     if self.netcat:
         paths = [str(i) for i in range(self.NC_RANGE)]
         time.sleep(self.SLEEPTIME)
     else:
         tuplpaths = get_pathlist(
             self.rsyncpath,
             self.rsyncdepth,
             exclude_re=self.excludere,
             # By default don't exclude user files
             exclude_usr=self.excl_usr,
             rsubpaths=self.rsubpaths)
         paths = encode_paths(tuplpaths)
     self.paths_total = len(paths)
     for path in paths:
         self.path_queue.put(path.encode(
         ))  # Put_all can issue a zookeeper connection error with big lists
     self.log.info('pathqueue building finished')
     return self.paths_total
Beispiel #6
0
    def test_get_pathlist_with_subpaths(self):
        """ Tests the functionality of get_pathlist with an exclude rule and subpaths"""
        regex = re.compile('/\.snapshots(/.*|$)')
        res = [
            ('/tree', 0),
            ('/tree/c1', 0),
            ('/tree/b1', 0),
            ('/tree/a1', 0),
            ('/tree/b1/bb2', 0),
            ('/tree/b1/ba2', 0),
            ('/tree/a1/ac2', 0),
            ('/tree/a1/ab2', 0),
            ('/tree/a1/aa2', 0),
            ('/tree/a1/ab2/aa3', 0),
            ('/tree/a1/ab2/aa3/sub1', 0),
            ('/tree/a1/ab2/aa3/sub2', 0),
            ('/tree/a1/ab2/aa3/sub2/sub21', 0),
            ('/tree/a1/ab2/aa3/sub2/sub21/sub211', 1),
        ]

        subpath1 = '3_a1/ab2/aa3'
        subpath2 = '3_a1/ab2/aa3/sub2'
        self.assertRaises(Exception,
                          dw.get_pathlist,
                          self.basedir,
                          3,
                          exclude_re=regex,
                          exclude_usr=None,
                          rsubpaths=['3_/a1/ab2/aa3'])
        self.assertRaises(Exception,
                          dw.get_pathlist,
                          self.basedir,
                          2,
                          exclude_re=regex,
                          exclude_usr=None,
                          rsubpaths=['3_a1/ab2/aa3'])
        self.assertRaises(Exception,
                          dw.get_pathlist,
                          self.basedir,
                          3,
                          exclude_re=regex,
                          exclude_usr=None,
                          rsubpaths=['1_a1'])

        genlist = [(re.sub('(/var)?/tmp/[^/]+', '/tree', gen), rec)
                   for (gen, rec) in dw.get_pathlist(
                       self.basedir, 3, exclude_re=regex, rsubpaths=[subpath1])
                   ]
        self.assertEqual(sorted(genlist), sorted(res))

        genlist = [(re.sub('(/var)?/tmp/[^/]+', '/tree', gen), rec) for (
            gen, rec
        ) in dw.get_pathlist(
            self.basedir, 3, exclude_re=regex, rsubpaths=[subpath1, subpath2])]
        res.remove(('/tree/a1/ab2/aa3/sub2/sub21/sub211', 1))
        res.extend([('/tree/a1/ab2/aa3/sub2/sub21/sub211', 0),
                    ('/tree/a1/ab2/aa3/sub2/sub21/sub211/sub2112', 1)])
        self.assertEqual(sorted(genlist), sorted(res))

        subpath2 = '4_a1/ab2/aa3/sub2'
        genlist = [(re.sub('(/var)?/tmp/[^/]+', '/tree', gen), rec) for (
            gen, rec
        ) in dw.get_pathlist(
            self.basedir, 3, exclude_re=regex, rsubpaths=[subpath1, subpath2])]
        res.remove(('/tree/a1/ab2/aa3/sub2/sub21/sub211/sub2112', 1))
        res.append(('/tree/a1/ab2/aa3/sub2/sub21/sub211/sub2112', 0))
        self.assertEqual(sorted(genlist), sorted(res))

        subpath1 = '3_a1/ab2'
        subpath2 = '3_a1/ab2/aa3'
        self.assertRaises(Exception,
                          dw.get_pathlist,
                          self.basedir,
                          3,
                          exclude_re=regex,
                          exclude_usr=None,
                          rsubpaths=[subpath2, subpath1])