示例#1
0
    def test_migrate_with_delayed_jobs(self):
        self.bs1.client.use('some-tube')
        for i in range(100):
            self.bs1.client.put_job(str(i))
        for i in range(100):
            self.bs1.client.put_job(str(i), delay=1000)

        assert self.bs1.status()['current-jobs-ready'] == 100
        assert self.bs1.status()['current-jobs-delayed'] == 100
        assert self.bs2.status()['current-jobs-ready'] == 0
        assert self.bs2.status()['current-jobs-delayed'] == 0

        parser = migrate.setup_parser()
        args = parser.parse_args([
            '-sh', self.bs1.host, '-sp', str(self.bs1.port),
            '-dh', self.bs2.host, '-dp', str(self.bs2.port),
            '-q',
            'some-tube'
        ])
        migrate.run(args)

        assert self.bs1.status()['current-jobs-ready'] == 0
        assert self.bs1.status()['current-jobs-delayed'] == 0
        assert self.bs2.status()['current-jobs-ready'] == 100
        assert self.bs2.status()['current-jobs-delayed'] == 100
示例#2
0
    def test_migrate_does_not_break_already_reserved_jobs(self):
        self.bs1.client.use('some-tube')
        for i in range(100):
            self.bs1.client.put_job(str(i), delay=0)
        self.bs1.client.watch('some-tube')
        job = self.bs1.client.reserve_job()

        assert self.bs1.status()['current-jobs-ready'] == 99
        assert self.bs1.status()['current-jobs-reserved'] == 1
        assert self.bs2.status()['current-jobs-ready'] == 0

        parser = migrate.setup_parser()
        args = parser.parse_args([
            '-sh', self.bs1.host, '-sp', str(self.bs1.port),
            '-dh', self.bs2.host, '-dp', str(self.bs2.port),
            '-q',
            'some-tube'
        ])
        migrate.run(args)

        assert self.bs1.status()['current-jobs-ready'] == 0
        assert self.bs1.status()['current-jobs-reserved'] == 1
        assert self.bs2.status()['current-jobs-ready'] == 99

        self.bs1.client.release_job(job.job_id)
        assert self.bs1.status()['current-jobs-ready'] == 1
        assert self.bs2.status()['current-jobs-ready'] == 99
示例#3
0
    def test_migrate_with_log(self):
        job_data = b'lots of data might contain invalid unicode, like \xad'
        b64_data = base64.b64encode(job_data).decode('ascii')
        self.generate_jobs(self.bs1.client, 'some-tube', ready=2, data=job_data)

        assert self.bs1.status()['current-jobs-ready'] == 2

        log_file = os.path.join(self.wd, 'migration_log')
        parser = migrate.setup_parser()
        args = parser.parse_args([
            '-sh', self.bs1.host, '-sp', str(self.bs1.port),
            '-dh', self.bs2.host, '-dp', str(self.bs2.port),
            '-q', '-l', log_file,
            'some-tube',
        ])
        migrate.run(args)

        assert self.bs1.status()['current-jobs-ready'] == 0

        with open(log_file, 'r') as log_f:
            lines = [json.loads(f.strip()) for f in log_f.readlines()]
            assert lines == [
                {'tube': 'some-tube', 'delay': 0, 'ttr': 120, 'job_data': b64_data, 'pri': 65535},
                {'tube': 'some-tube', 'delay': 0, 'ttr': 120, 'job_data': b64_data, 'pri': 65535},
            ]
示例#4
0
    def test_ignores_buried_jobs_with_B(self):
        self.bs1.client.put_job('1', delay=0)
        self.bs1.client.put_job('2', delay=0)
        job = self.bs1.client.reserve_job()
        self.bs1.client.bury_job(job.job_id)

        assert self.bs1.status()['current-jobs-ready'] == 1
        assert self.bs1.status()['current-jobs-buried'] == 1
        assert self.bs1.status()['current-jobs-reserved'] == 0

        parser = migrate.setup_parser()
        args = parser.parse_args([
            '-sh', self.bs1.host, '-sp', str(self.bs1.port),
            '-dh', self.bs2.host, '-dp', str(self.bs2.port),
            '-q', '-B',
            'default'
        ])
        migrate.run(args)

        assert self.bs1.status()['current-jobs-ready'] == 0
        assert self.bs1.status()['current-jobs-buried'] == 1
        assert self.bs1.status()['current-jobs-reserved'] == 0

        assert self.bs2.status()['current-jobs-ready'] == 1
        assert self.bs2.status()['current-jobs-buried'] == 0
        assert self.bs2.status()['current-jobs-reserved'] == 0
示例#5
0
    def test_migrates_buried_jobs_back_to_runnable(self):
        self.bs1.client.use('some-tube')
        self.bs1.client.put_job('1', delay=0)
        self.bs1.client.put_job('2', delay=0)
        self.bs1.client.watch('some-tube')
        job = self.bs1.client.reserve_job()
        self.bs1.client.bury_job(job.job_id)

        assert self.bs1.status()['current-jobs-ready'] == 1
        assert self.bs1.status()['current-jobs-buried'] == 1
        assert self.bs1.status()['current-jobs-reserved'] == 0

        parser = migrate.setup_parser()
        args = parser.parse_args([
            '-sh', self.bs1.host, '-sp', str(self.bs1.port),
            '-dh', self.bs2.host, '-dp', str(self.bs2.port),
            '-q',
            'some-tube'
        ])
        migrate.run(args)

        assert self.bs1.status()['current-jobs-ready'] == 0
        assert self.bs1.status()['current-jobs-buried'] == 0
        assert self.bs1.status()['current-jobs-reserved'] == 0

        assert self.bs2.status()['current-jobs-ready'] == 2
        assert self.bs2.status()['current-jobs-buried'] == 0
        assert self.bs2.status()['current-jobs-reserved'] == 0
示例#6
0
    def test_migrate_and_replay(self):
        # create a bunch of jobs
        self.bs1.client.use('tube1')
        for i in range(200):
            self.bs1.client.put_job(str(i))

        log_file_path = os.path.join(self.wd, 'log')

        # migrate them into bs2, keeping a log
        assert self.bs1.client.stats()['current-jobs-ready'] == 200
        parser = migrate.setup_parser()
        args = parser.parse_args([
            '-sh', self.bs1.host, '-sp', str(self.bs1.port),
            '-dh', self.bs2.host, '-dp', str(self.bs2.port),
            '-q', '-l', log_file_path,
            'tube1'
        ])
        migrate.run(args)
        assert self.bs1.client.stats()['current-jobs-ready'] == 0
        assert self.bs2.client.stats()['current-jobs-ready'] == 200

        # replay the log back into bs1
        parser = replay.setup_parser()
        args = parser.parse_args([
            '-H', self.bs1.host, '-p', str(self.bs1.port),
            log_file_path,
        ])
        replay.run(args)
        assert self.bs1.client.stats()['current-jobs-ready'] == 200
示例#7
0
    def test_migrate_multiple_tubes(self):
        self.generate_jobs(self.bs1.client, 'some-tube', 2)
        self.generate_jobs(self.bs1.client, 'some-other-tube', 2)
        self.generate_jobs(self.bs1.client, 'ignored-tube', 1)

        assert self.bs1.status()['current-jobs-ready'] == 5

        parser = migrate.setup_parser()
        args = parser.parse_args([
            '-sh', self.bs1.host, '-sp', str(self.bs1.port),
            '-dh', self.bs2.host, '-dp', str(self.bs2.port),
            '-q',
            'some-tube', 'some-other-tube'
        ])
        migrate.run(args)

        assert self.bs1.status()['current-jobs-ready'] == 1
        assert self.bs2.status()['current-jobs-ready'] == 4

        assert self.bs1.client.stats_tube('ignored-tube')['current-jobs-ready'] == 1
示例#8
0
    def test_migrate_rename_tube(self):
        self.generate_jobs(self.bs1.client, 'some-tube', ready=1, delayed=1)

        assert self.bs1.client.stats_tube('some-tube')['current-jobs-ready'] == 1
        assert self.bs1.client.stats_tube('some-tube')['current-jobs-delayed'] == 1

        parser = migrate.setup_parser()
        args = parser.parse_args([
            '-sh', self.bs1.host, '-sp', str(self.bs1.port),
            '-dh', self.bs2.host, '-dp', str(self.bs2.port),
            '-T', 'some-other-tube', '-q',
            'some-tube',
        ])
        migrate.run(args)

        assert self.bs1.client.stats_tube('some-tube')['current-jobs-ready'] == 0
        assert self.bs1.client.stats_tube('some-tube')['current-jobs-delayed'] == 0
        with pytest.raises(pystalk.BeanstalkError):
            assert self.bs2.client.stats_tube('some-tube')['current-jobs-ready'] == 0
        with pytest.raises(pystalk.BeanstalkError):
            assert self.bs2.client.stats_tube('some-tube')['current-jobs-delayed'] == 0
        assert self.bs2.client.stats_tube('some-other-tube')['current-jobs-ready'] == 1
        assert self.bs2.client.stats_tube('some-other-tube')['current-jobs-delayed'] == 1
示例#9
0
    def test_verbose_output(self):
        self.generate_jobs(self.bs1.client, 'some-tube', ready=1, delayed=1, buried=1)

        assert self.bs1.client.stats_tube('some-tube')['current-jobs-ready'] == 1
        assert self.bs1.client.stats_tube('some-tube')['current-jobs-delayed'] == 1
        assert self.bs1.client.stats_tube('some-tube')['current-jobs-buried'] == 1

        parser = migrate.setup_parser()
        args = parser.parse_args([
            '-sh', self.bs1.host, '-sp', str(self.bs1.port),
            '-dh', self.bs2.host, '-dp', str(self.bs2.port),
            'some-tube',
        ])

        with mock.patch('sys.stderr.write') as mock_stderr:
            with mock.patch('sys.stdout.write') as mock_stdout:
                migrate.run(args)
        # stdout should never be written to
        assert mock_stdout.mock_calls == []
        mock_stderr.assert_has_calls([
            mock.call('Migrating ~1 READY jobs on tube some-tube'),
            mock.call('Migrating ~1 DELAYED jobs on tube some-tube'),
            mock.call('Migrating ~1 BURIED jobs on tube some-tube'),
        ], any_order=True)