def test_migrate_with_delayed_jobs(self): self.bs1.client.use('some-tube') for i in range(100): self.bs1.client.put_job(str(i)) for i in range(100): self.bs1.client.put_job(str(i), delay=1000) assert self.bs1.status()['current-jobs-ready'] == 100 assert self.bs1.status()['current-jobs-delayed'] == 100 assert self.bs2.status()['current-jobs-ready'] == 0 assert self.bs2.status()['current-jobs-delayed'] == 0 parser = migrate.setup_parser() args = parser.parse_args([ '-sh', self.bs1.host, '-sp', str(self.bs1.port), '-dh', self.bs2.host, '-dp', str(self.bs2.port), '-q', 'some-tube' ]) migrate.run(args) assert self.bs1.status()['current-jobs-ready'] == 0 assert self.bs1.status()['current-jobs-delayed'] == 0 assert self.bs2.status()['current-jobs-ready'] == 100 assert self.bs2.status()['current-jobs-delayed'] == 100
def test_migrate_does_not_break_already_reserved_jobs(self): self.bs1.client.use('some-tube') for i in range(100): self.bs1.client.put_job(str(i), delay=0) self.bs1.client.watch('some-tube') job = self.bs1.client.reserve_job() assert self.bs1.status()['current-jobs-ready'] == 99 assert self.bs1.status()['current-jobs-reserved'] == 1 assert self.bs2.status()['current-jobs-ready'] == 0 parser = migrate.setup_parser() args = parser.parse_args([ '-sh', self.bs1.host, '-sp', str(self.bs1.port), '-dh', self.bs2.host, '-dp', str(self.bs2.port), '-q', 'some-tube' ]) migrate.run(args) assert self.bs1.status()['current-jobs-ready'] == 0 assert self.bs1.status()['current-jobs-reserved'] == 1 assert self.bs2.status()['current-jobs-ready'] == 99 self.bs1.client.release_job(job.job_id) assert self.bs1.status()['current-jobs-ready'] == 1 assert self.bs2.status()['current-jobs-ready'] == 99
def test_migrate_with_log(self): job_data = b'lots of data might contain invalid unicode, like \xad' b64_data = base64.b64encode(job_data).decode('ascii') self.generate_jobs(self.bs1.client, 'some-tube', ready=2, data=job_data) assert self.bs1.status()['current-jobs-ready'] == 2 log_file = os.path.join(self.wd, 'migration_log') parser = migrate.setup_parser() args = parser.parse_args([ '-sh', self.bs1.host, '-sp', str(self.bs1.port), '-dh', self.bs2.host, '-dp', str(self.bs2.port), '-q', '-l', log_file, 'some-tube', ]) migrate.run(args) assert self.bs1.status()['current-jobs-ready'] == 0 with open(log_file, 'r') as log_f: lines = [json.loads(f.strip()) for f in log_f.readlines()] assert lines == [ {'tube': 'some-tube', 'delay': 0, 'ttr': 120, 'job_data': b64_data, 'pri': 65535}, {'tube': 'some-tube', 'delay': 0, 'ttr': 120, 'job_data': b64_data, 'pri': 65535}, ]
def test_ignores_buried_jobs_with_B(self): self.bs1.client.put_job('1', delay=0) self.bs1.client.put_job('2', delay=0) job = self.bs1.client.reserve_job() self.bs1.client.bury_job(job.job_id) assert self.bs1.status()['current-jobs-ready'] == 1 assert self.bs1.status()['current-jobs-buried'] == 1 assert self.bs1.status()['current-jobs-reserved'] == 0 parser = migrate.setup_parser() args = parser.parse_args([ '-sh', self.bs1.host, '-sp', str(self.bs1.port), '-dh', self.bs2.host, '-dp', str(self.bs2.port), '-q', '-B', 'default' ]) migrate.run(args) assert self.bs1.status()['current-jobs-ready'] == 0 assert self.bs1.status()['current-jobs-buried'] == 1 assert self.bs1.status()['current-jobs-reserved'] == 0 assert self.bs2.status()['current-jobs-ready'] == 1 assert self.bs2.status()['current-jobs-buried'] == 0 assert self.bs2.status()['current-jobs-reserved'] == 0
def test_migrates_buried_jobs_back_to_runnable(self): self.bs1.client.use('some-tube') self.bs1.client.put_job('1', delay=0) self.bs1.client.put_job('2', delay=0) self.bs1.client.watch('some-tube') job = self.bs1.client.reserve_job() self.bs1.client.bury_job(job.job_id) assert self.bs1.status()['current-jobs-ready'] == 1 assert self.bs1.status()['current-jobs-buried'] == 1 assert self.bs1.status()['current-jobs-reserved'] == 0 parser = migrate.setup_parser() args = parser.parse_args([ '-sh', self.bs1.host, '-sp', str(self.bs1.port), '-dh', self.bs2.host, '-dp', str(self.bs2.port), '-q', 'some-tube' ]) migrate.run(args) assert self.bs1.status()['current-jobs-ready'] == 0 assert self.bs1.status()['current-jobs-buried'] == 0 assert self.bs1.status()['current-jobs-reserved'] == 0 assert self.bs2.status()['current-jobs-ready'] == 2 assert self.bs2.status()['current-jobs-buried'] == 0 assert self.bs2.status()['current-jobs-reserved'] == 0
def test_migrate_and_replay(self): # create a bunch of jobs self.bs1.client.use('tube1') for i in range(200): self.bs1.client.put_job(str(i)) log_file_path = os.path.join(self.wd, 'log') # migrate them into bs2, keeping a log assert self.bs1.client.stats()['current-jobs-ready'] == 200 parser = migrate.setup_parser() args = parser.parse_args([ '-sh', self.bs1.host, '-sp', str(self.bs1.port), '-dh', self.bs2.host, '-dp', str(self.bs2.port), '-q', '-l', log_file_path, 'tube1' ]) migrate.run(args) assert self.bs1.client.stats()['current-jobs-ready'] == 0 assert self.bs2.client.stats()['current-jobs-ready'] == 200 # replay the log back into bs1 parser = replay.setup_parser() args = parser.parse_args([ '-H', self.bs1.host, '-p', str(self.bs1.port), log_file_path, ]) replay.run(args) assert self.bs1.client.stats()['current-jobs-ready'] == 200
def test_migrate_multiple_tubes(self): self.generate_jobs(self.bs1.client, 'some-tube', 2) self.generate_jobs(self.bs1.client, 'some-other-tube', 2) self.generate_jobs(self.bs1.client, 'ignored-tube', 1) assert self.bs1.status()['current-jobs-ready'] == 5 parser = migrate.setup_parser() args = parser.parse_args([ '-sh', self.bs1.host, '-sp', str(self.bs1.port), '-dh', self.bs2.host, '-dp', str(self.bs2.port), '-q', 'some-tube', 'some-other-tube' ]) migrate.run(args) assert self.bs1.status()['current-jobs-ready'] == 1 assert self.bs2.status()['current-jobs-ready'] == 4 assert self.bs1.client.stats_tube('ignored-tube')['current-jobs-ready'] == 1
def test_migrate_rename_tube(self): self.generate_jobs(self.bs1.client, 'some-tube', ready=1, delayed=1) assert self.bs1.client.stats_tube('some-tube')['current-jobs-ready'] == 1 assert self.bs1.client.stats_tube('some-tube')['current-jobs-delayed'] == 1 parser = migrate.setup_parser() args = parser.parse_args([ '-sh', self.bs1.host, '-sp', str(self.bs1.port), '-dh', self.bs2.host, '-dp', str(self.bs2.port), '-T', 'some-other-tube', '-q', 'some-tube', ]) migrate.run(args) assert self.bs1.client.stats_tube('some-tube')['current-jobs-ready'] == 0 assert self.bs1.client.stats_tube('some-tube')['current-jobs-delayed'] == 0 with pytest.raises(pystalk.BeanstalkError): assert self.bs2.client.stats_tube('some-tube')['current-jobs-ready'] == 0 with pytest.raises(pystalk.BeanstalkError): assert self.bs2.client.stats_tube('some-tube')['current-jobs-delayed'] == 0 assert self.bs2.client.stats_tube('some-other-tube')['current-jobs-ready'] == 1 assert self.bs2.client.stats_tube('some-other-tube')['current-jobs-delayed'] == 1
def test_verbose_output(self): self.generate_jobs(self.bs1.client, 'some-tube', ready=1, delayed=1, buried=1) assert self.bs1.client.stats_tube('some-tube')['current-jobs-ready'] == 1 assert self.bs1.client.stats_tube('some-tube')['current-jobs-delayed'] == 1 assert self.bs1.client.stats_tube('some-tube')['current-jobs-buried'] == 1 parser = migrate.setup_parser() args = parser.parse_args([ '-sh', self.bs1.host, '-sp', str(self.bs1.port), '-dh', self.bs2.host, '-dp', str(self.bs2.port), 'some-tube', ]) with mock.patch('sys.stderr.write') as mock_stderr: with mock.patch('sys.stdout.write') as mock_stdout: migrate.run(args) # stdout should never be written to assert mock_stdout.mock_calls == [] mock_stderr.assert_has_calls([ mock.call('Migrating ~1 READY jobs on tube some-tube'), mock.call('Migrating ~1 DELAYED jobs on tube some-tube'), mock.call('Migrating ~1 BURIED jobs on tube some-tube'), ], any_order=True)