def test_peek_buried(c: Client) -> None: id = c.put(b"buried") job = c.reserve() c.bury(job) job = c.peek_buried() assert job.id == id assert job.body == b"buried"
def test_put_priority(c: Client) -> None: c.put('2', priority=2) c.put('1', priority=1) job = c.reserve() assert job.body == '1' job = c.reserve() assert job.body == '2'
def test_str_body_no_encoding() -> None: class Fake: def __init__(self) -> None: self.encoding = None with pytest.raises(TypeError): Client.put(Fake(), 'a str job') # type: ignore
def test_put_priority(c: Client) -> None: c.put(b"2", priority=2) c.put(b"1", priority=1) job = c.reserve() assert job.body == b"1" job = c.reserve() assert job.body == b"2"
def test_kick(c: Client) -> None: c.put(b"a delayed job", delay=30) c.put(b"another delayed job", delay=45) c.put(b"a ready job") job = c.reserve() c.bury(job) assert c.kick(10) == 1 assert c.kick(10) == 2
def test_basic_usage(c: Client) -> None: c.use('emails') id = c.put('测试@example.com') c.watch('emails') c.ignore('default') job = c.reserve() assert id == job.id assert job.body == '测试@example.com' c.delete(job)
def test_basic_usage(c: Client) -> None: c.use("emails") id = c.put("测试@example.com".encode("utf-8")) c.watch("emails") c.ignore("default") job = c.reserve() assert id == job.id assert job.body.decode("utf-8") == "测试@example.com" c.delete(job)
def test_delete_job_reserved_by_other(c: Client) -> None: c.put('', ttr=1) other = Client(port=PORT) job = other.reserve() with pytest.raises(NotFoundError): c.delete(job) time.sleep(1) c.delete(job)
def max_current_jobs_ready(client: greenstalk.Client, tubes: List[Text]) -> int: """Get max current-jobs-ready from tubes specified""" max_jobs = 0 for tube in tubes: # We need to check for the existense of a beanstalk tube # to avoid "NOT_FOUND" exceptions in the case where the tubes # are empty before first init if tube not in client.tubes(): continue stats = client.stats_tube(tube) if stats: max_jobs = max(max_jobs, stats.get("current-jobs-ready", 0)) return max_jobs
def main(): init_logging() init_model(config.SQLALCHEMY_URL) shutdown_event = threading.Event() scheduler = BackgroundScheduler() # workflow_publisher = configured_publisher() activity_sync = ActivitySync() weather_sync = WeatherSync() athlete_sync = AthleteSync() # Every hour run a sync on the activities for athletes fall into the specified segment # athlete_id % total_segments == segment # TODO: Probably it would be more prudent to split into 15-minute segments, to match rate limits # Admittedly that will make the time-based segment calculation a little trickier. def segmented_sync_activities(): activity_sync.sync_rides_distributed(total_segments=4, segment=(arrow.now().hour % 4)) scheduler.add_job(segmented_sync_activities, 'cron', minute='50') # This should generally not pick up anytihng. scheduler.add_job(activity_sync.sync_rides_detail, 'cron', minute='20') # Sync weather at 8am UTC scheduler.add_job(weather_sync.sync_weather, 'cron', hour='8') # Sync athletes once a day at 6am UTC scheduler.add_job(athlete_sync.sync_athletes, 'cron', minute='30') scheduler.start() beanclient = Client(host=config.BEANSTALKD_HOST, port=config.BEANSTALKD_PORT, watch=[DefinedTubes.activity_update.value]) subscriber = ActivityUpdateSubscriber(beanstalk_client=beanclient, shutdown_event=shutdown_event) def shutdown_app(): shutdown_event.wait() scheduler.shutdown() shutdown_monitor = threading.Thread(target=shutdown_app) shutdown_monitor.start() try: # This is here to simulate application activity (which keeps the main thread alive). subscriber.run_forever() except (KeyboardInterrupt, SystemExit): log.info("Exiting on user request.") except: log.exception("Error running sync/listener.") finally: shutdown_event.set() shutdown_monitor.join()
def test_stats_job(c: Client) -> None: assert c.stats_job(c.put('job')) == { 'id': 1, 'tube': 'default', 'state': 'ready', 'pri': DEFAULT_PRIORITY, 'age': 0, 'delay': 0, 'ttr': DEFAULT_TTR, 'time-left': 0, 'file': 0, 'reserves': 0, 'timeouts': 0, 'releases': 0, 'buries': 0, 'kicks': 0, }
def test_stats_job(c: Client) -> None: assert c.stats_job(c.put(b"job")) == { "id": 1, "tube": "default", "state": "ready", "pri": DEFAULT_PRIORITY, "age": 0, "delay": 0, "ttr": DEFAULT_TTR, "time-left": 0, "file": 0, "reserves": 0, "timeouts": 0, "releases": 0, "buries": 0, "kicks": 0, }
def test_tubes(c: Client) -> None: assert c.tubes() == ['default'] c.use('a') assert set(c.tubes()) == {'default', 'a'} c.watch('b') c.watch('c') assert set(c.tubes()) == {'default', 'a', 'b', 'c'}
def test_tubes(c: Client) -> None: assert c.tubes() == ["default"] c.use("a") assert set(c.tubes()) == {"default", "a"} c.watch("b") c.watch("c") assert set(c.tubes()) == {"default", "a", "b", "c"}
async def wrapper() -> None: cmd = ('beanstalkd', '-l', '127.0.0.1', '-p', str(PORT)) with subprocess.Popen(cmd) as beanstalkd: time.sleep(0.1) try: async with Client(port=PORT, **kwargs) as c: await test(c) finally: beanstalkd.terminate()
def wrapper() -> None: port = 4444 cmd = ('beanstalkd', '-l', '127.0.0.1', '-p', str(port)) with subprocess.Popen(cmd) as beanstalkd: time.sleep(0.1) try: with Client(port=port, **kwargs) as c: test(c) finally: beanstalkd.terminate()
def test_stats(c: Client) -> None: s = c.stats() assert s["current-jobs-urgent"] == 0 assert s["current-jobs-ready"] == 0 assert s["current-jobs-reserved"] == 0 assert s["current-jobs-delayed"] == 0 assert s["current-jobs-buried"] == 0 assert s["cmd-put"] == 0 assert s["cmd-peek"] == 0 assert s["cmd-peek-ready"] == 0 assert s["cmd-peek-delayed"] == 0 assert s["cmd-peek-buried"] == 0 assert s["cmd-reserve"] == 0 assert s["cmd-reserve-with-timeout"] == 0 assert s["cmd-delete"] == 0 assert s["cmd-release"] == 0 assert s["cmd-use"] == 0 assert s["cmd-watch"] == 0 assert s["cmd-ignore"] == 0 assert s["cmd-bury"] == 0 assert s["cmd-kick"] == 0 assert s["cmd-touch"] == 0 assert s["cmd-stats"] == 1 assert s["cmd-stats-job"] == 0 assert s["cmd-stats-tube"] == 0 assert s["cmd-list-tubes"] == 0 assert s["cmd-list-tube-used"] == 0 assert s["cmd-list-tubes-watched"] == 0 assert s["cmd-pause-tube"] == 0 assert s["job-timeouts"] == 0 assert s["total-jobs"] == 0 assert "max-job-size" in s assert s["current-tubes"] == 1 assert s["current-connections"] == 1 assert s["current-producers"] == 0 assert s["current-workers"] == 0 assert s["current-waiting"] == 0 assert s["total-connections"] == 1 assert "pid" in s assert "version" in s assert "rusage-utime" in s assert "rusage-stime" in s assert s["uptime"] == 0 assert s["binlog-oldest-index"] == 0 assert s["binlog-current-index"] == 0 assert s["binlog-records-migrated"] == 0 assert s["binlog-records-written"] == 0 assert "binlog-max-size" in s assert "id" in s assert "hostname" in s assert s["draining"] == "false"
def test_stats(c: Client) -> None: s = c.stats() assert s['current-jobs-urgent'] == 0 assert s['current-jobs-ready'] == 0 assert s['current-jobs-reserved'] == 0 assert s['current-jobs-delayed'] == 0 assert s['current-jobs-buried'] == 0 assert s['cmd-put'] == 0 assert s['cmd-peek'] == 0 assert s['cmd-peek-ready'] == 0 assert s['cmd-peek-delayed'] == 0 assert s['cmd-peek-buried'] == 0 assert s['cmd-reserve'] == 0 assert s['cmd-reserve-with-timeout'] == 0 assert s['cmd-delete'] == 0 assert s['cmd-release'] == 0 assert s['cmd-use'] == 0 assert s['cmd-watch'] == 0 assert s['cmd-ignore'] == 0 assert s['cmd-bury'] == 0 assert s['cmd-kick'] == 0 assert s['cmd-touch'] == 0 assert s['cmd-stats'] == 1 assert s['cmd-stats-job'] == 0 assert s['cmd-stats-tube'] == 0 assert s['cmd-list-tubes'] == 0 assert s['cmd-list-tube-used'] == 0 assert s['cmd-list-tubes-watched'] == 0 assert s['cmd-pause-tube'] == 0 assert s['job-timeouts'] == 0 assert s['total-jobs'] == 0 assert 'max-job-size' in s assert s['current-tubes'] == 1 assert s['current-connections'] == 1 assert s['current-producers'] == 0 assert s['current-workers'] == 0 assert s['current-waiting'] == 0 assert s['total-connections'] == 1 assert 'pid' in s assert 'version' in s assert 'rusage-utime' in s assert 'rusage-stime' in s assert s['uptime'] == 0 assert s['binlog-oldest-index'] == 0 assert s['binlog-current-index'] == 0 assert s['binlog-records-migrated'] == 0 assert s['binlog-records-written'] == 0 assert 'binlog-max-size' in s assert 'id' in s assert 'hostname' in s
def wrapper() -> None: cmd = [BEANSTALKD_PATH] if isinstance(address, str): cmd.extend(["-l", "unix:" + address]) else: host, port = address cmd.extend(["-l", host, "-p", str(port)]) with subprocess.Popen(cmd) as beanstalkd: time.sleep(0.1) try: with Client(address, use=use, watch=watch) as c: test(c) finally: beanstalkd.terminate()
def test_drain_mode() -> None: cmd = [BEANSTALKD_PATH, "-l", "unix:" + DEFAULT_UNIX_ADDRESS] with subprocess.Popen(cmd) as beanstalkd: time.sleep(0.1) try: with Client(address=DEFAULT_UNIX_ADDRESS) as c: assert c.put(b"first") == 1 beanstalkd.send_signal(signal.SIGUSR1) time.sleep(0.1) with pytest.raises(DrainingError): c.put(b"second") assert c.stats()["draining"] == "true" finally: beanstalkd.terminate()
def wrapper() -> None: cmd = [BEANSTALKD_PATH] if isinstance(address, str): cmd.extend(['-l', 'unix:' + address]) else: host, port = address cmd.extend(['-l', host, '-p', str(port)]) with subprocess.Popen(cmd) as beanstalkd: time.sleep(0.1) try: with Client(address, **kwargs) as c: test(c) finally: beanstalkd.terminate()
def test_stats_tube(c: Client) -> None: assert c.stats_tube('default') == { 'name': 'default', 'current-jobs-urgent': 0, 'current-jobs-ready': 0, 'current-jobs-reserved': 0, 'current-jobs-delayed': 0, 'current-jobs-buried': 0, 'total-jobs': 0, 'current-using': 0, 'current-watching': 1, 'current-waiting': 0, 'cmd-delete': 0, 'cmd-pause-tube': 0, 'pause': 0, 'pause-time-left': 0, }
def test_stats_tube(c: Client) -> None: assert c.stats_tube("default") == { "name": "default", "current-jobs-urgent": 0, "current-jobs-ready": 0, "current-jobs-reserved": 0, "current-jobs-delayed": 0, "current-jobs-buried": 0, "total-jobs": 0, "current-using": 0, "current-watching": 1, "current-waiting": 0, "cmd-delete": 0, "cmd-pause-tube": 0, "pause": 0, "pause-time-left": 0, }
def test_delays(c: Client) -> None: with assert_seconds(1): c.put(b"delayed", delay=1) job = c.reserve() assert job.body == b"delayed" with assert_seconds(2): c.release(job, delay=2) with pytest.raises(TimedOutError): c.reserve(timeout=1) job = c.reserve(timeout=1) c.bury(job) with pytest.raises(TimedOutError): c.reserve(timeout=0)
def test_not_ignored(c: Client) -> None: with pytest.raises(NotIgnoredError): c.ignore("default")
def test_delete_job_reserved_by_other(c: Client) -> None: c.put(b"", ttr=1) with Client(DEFAULT_INET_ADDRESS) as other: job = other.reserve() with pytest.raises(NotFoundError): c.delete(job)
def test_job_not_found(c: Client) -> None: with pytest.raises(NotFoundError): c.delete(87)
def test_max_job_size(c: Client) -> None: with pytest.raises(JobTooBigError): c.put(bytes(2 ** 16))
def test_pause_tube(c: Client) -> None: c.put(b"") with assert_seconds(1): c.pause_tube("default", 1) c.reserve()
def test_watching(c: Client) -> None: assert c.watching() == ["default"] c.watch("another") assert set(c.watching()) == {"default", "another"}