def test_peek_buried(c: Client) -> None:
id = c.put(b"buried")
job = c.reserve()
c.bury(job)
job = c.peek_buried()
assert job.id == id
assert job.body == b"buried"
def test_put_priority(c: Client) -> None:
c.put('2', priority=2)
c.put('1', priority=1)
job = c.reserve()
assert job.body == '1'
job = c.reserve()
assert job.body == '2'
def test_str_body_no_encoding() -> None:
class Fake:
def __init__(self) -> None:
self.encoding = None
with pytest.raises(TypeError):
Client.put(Fake(), 'a str job') # type: ignore
def test_put_priority(c: Client) -> None:
c.put(b"2", priority=2)
c.put(b"1", priority=1)
job = c.reserve()
assert job.body == b"1"
job = c.reserve()
assert job.body == b"2"
def test_kick(c: Client) -> None:
c.put(b"a delayed job", delay=30)
c.put(b"another delayed job", delay=45)
c.put(b"a ready job")
job = c.reserve()
c.bury(job)
assert c.kick(10) == 1
assert c.kick(10) == 2
def test_basic_usage(c: Client) -> None:
c.use('emails')
id = c.put('测试@example.com')
c.watch('emails')
c.ignore('default')
job = c.reserve()
assert id == job.id
assert job.body == '测试@example.com'
c.delete(job)
def test_basic_usage(c: Client) -> None:
c.use("emails")
id = c.put("测试@example.com".encode("utf-8"))
c.watch("emails")
c.ignore("default")
job = c.reserve()
assert id == job.id
assert job.body.decode("utf-8") == "测试@example.com"
c.delete(job)
def test_delete_job_reserved_by_other(c: Client) -> None:
c.put('', ttr=1)
other = Client(port=PORT)
job = other.reserve()
with pytest.raises(NotFoundError):
c.delete(job)
time.sleep(1)
c.delete(job)
def max_current_jobs_ready(client: greenstalk.Client,
tubes: List[Text]) -> int:
"""Get max current-jobs-ready from tubes specified"""
max_jobs = 0
for tube in tubes:
# We need to check for the existense of a beanstalk tube
# to avoid "NOT_FOUND" exceptions in the case where the tubes
# are empty before first init
if tube not in client.tubes():
continue
stats = client.stats_tube(tube)
if stats:
max_jobs = max(max_jobs, stats.get("current-jobs-ready", 0))
return max_jobs
def main():
init_logging()
init_model(config.SQLALCHEMY_URL)
shutdown_event = threading.Event()
scheduler = BackgroundScheduler()
# workflow_publisher = configured_publisher()
activity_sync = ActivitySync()
weather_sync = WeatherSync()
athlete_sync = AthleteSync()
# Every hour run a sync on the activities for athletes fall into the specified segment
# athlete_id % total_segments == segment
# TODO: Probably it would be more prudent to split into 15-minute segments, to match rate limits
# Admittedly that will make the time-based segment calculation a little trickier.
def segmented_sync_activities():
activity_sync.sync_rides_distributed(total_segments=4,
segment=(arrow.now().hour % 4))
scheduler.add_job(segmented_sync_activities, 'cron', minute='50')
# This should generally not pick up anytihng.
scheduler.add_job(activity_sync.sync_rides_detail, 'cron', minute='20')
# Sync weather at 8am UTC
scheduler.add_job(weather_sync.sync_weather, 'cron', hour='8')
# Sync athletes once a day at 6am UTC
scheduler.add_job(athlete_sync.sync_athletes, 'cron', minute='30')
scheduler.start()
beanclient = Client(host=config.BEANSTALKD_HOST,
port=config.BEANSTALKD_PORT,
watch=[DefinedTubes.activity_update.value])
subscriber = ActivityUpdateSubscriber(beanstalk_client=beanclient,
shutdown_event=shutdown_event)
def shutdown_app():
shutdown_event.wait()
scheduler.shutdown()
shutdown_monitor = threading.Thread(target=shutdown_app)
shutdown_monitor.start()
try:
# This is here to simulate application activity (which keeps the main thread alive).
subscriber.run_forever()
except (KeyboardInterrupt, SystemExit):
log.info("Exiting on user request.")
except:
log.exception("Error running sync/listener.")
finally:
shutdown_event.set()
shutdown_monitor.join()
def test_stats_job(c: Client) -> None:
assert c.stats_job(c.put('job')) == {
'id': 1,
'tube': 'default',
'state': 'ready',
'pri': DEFAULT_PRIORITY,
'age': 0,
'delay': 0,
'ttr': DEFAULT_TTR,
'time-left': 0,
'file': 0,
'reserves': 0,
'timeouts': 0,
'releases': 0,
'buries': 0,
'kicks': 0,
}
def test_stats_job(c: Client) -> None:
assert c.stats_job(c.put(b"job")) == {
"id": 1,
"tube": "default",
"state": "ready",
"pri": DEFAULT_PRIORITY,
"age": 0,
"delay": 0,
"ttr": DEFAULT_TTR,
"time-left": 0,
"file": 0,
"reserves": 0,
"timeouts": 0,
"releases": 0,
"buries": 0,
"kicks": 0,
}
def test_tubes(c: Client) -> None:
assert c.tubes() == ['default']
c.use('a')
assert set(c.tubes()) == {'default', 'a'}
c.watch('b')
c.watch('c')
assert set(c.tubes()) == {'default', 'a', 'b', 'c'}
def test_tubes(c: Client) -> None:
assert c.tubes() == ["default"]
c.use("a")
assert set(c.tubes()) == {"default", "a"}
c.watch("b")
c.watch("c")
assert set(c.tubes()) == {"default", "a", "b", "c"}
async def wrapper() -> None:
cmd = ('beanstalkd', '-l', '127.0.0.1', '-p', str(PORT))
with subprocess.Popen(cmd) as beanstalkd:
time.sleep(0.1)
try:
async with Client(port=PORT, **kwargs) as c:
await test(c)
finally:
beanstalkd.terminate()
def wrapper() -> None:
port = 4444
cmd = ('beanstalkd', '-l', '127.0.0.1', '-p', str(port))
with subprocess.Popen(cmd) as beanstalkd:
time.sleep(0.1)
try:
with Client(port=port, **kwargs) as c:
test(c)
finally:
beanstalkd.terminate()
def test_stats(c: Client) -> None:
s = c.stats()
assert s["current-jobs-urgent"] == 0
assert s["current-jobs-ready"] == 0
assert s["current-jobs-reserved"] == 0
assert s["current-jobs-delayed"] == 0
assert s["current-jobs-buried"] == 0
assert s["cmd-put"] == 0
assert s["cmd-peek"] == 0
assert s["cmd-peek-ready"] == 0
assert s["cmd-peek-delayed"] == 0
assert s["cmd-peek-buried"] == 0
assert s["cmd-reserve"] == 0
assert s["cmd-reserve-with-timeout"] == 0
assert s["cmd-delete"] == 0
assert s["cmd-release"] == 0
assert s["cmd-use"] == 0
assert s["cmd-watch"] == 0
assert s["cmd-ignore"] == 0
assert s["cmd-bury"] == 0
assert s["cmd-kick"] == 0
assert s["cmd-touch"] == 0
assert s["cmd-stats"] == 1
assert s["cmd-stats-job"] == 0
assert s["cmd-stats-tube"] == 0
assert s["cmd-list-tubes"] == 0
assert s["cmd-list-tube-used"] == 0
assert s["cmd-list-tubes-watched"] == 0
assert s["cmd-pause-tube"] == 0
assert s["job-timeouts"] == 0
assert s["total-jobs"] == 0
assert "max-job-size" in s
assert s["current-tubes"] == 1
assert s["current-connections"] == 1
assert s["current-producers"] == 0
assert s["current-workers"] == 0
assert s["current-waiting"] == 0
assert s["total-connections"] == 1
assert "pid" in s
assert "version" in s
assert "rusage-utime" in s
assert "rusage-stime" in s
assert s["uptime"] == 0
assert s["binlog-oldest-index"] == 0
assert s["binlog-current-index"] == 0
assert s["binlog-records-migrated"] == 0
assert s["binlog-records-written"] == 0
assert "binlog-max-size" in s
assert "id" in s
assert "hostname" in s
assert s["draining"] == "false"
def test_stats(c: Client) -> None:
s = c.stats()
assert s['current-jobs-urgent'] == 0
assert s['current-jobs-ready'] == 0
assert s['current-jobs-reserved'] == 0
assert s['current-jobs-delayed'] == 0
assert s['current-jobs-buried'] == 0
assert s['cmd-put'] == 0
assert s['cmd-peek'] == 0
assert s['cmd-peek-ready'] == 0
assert s['cmd-peek-delayed'] == 0
assert s['cmd-peek-buried'] == 0
assert s['cmd-reserve'] == 0
assert s['cmd-reserve-with-timeout'] == 0
assert s['cmd-delete'] == 0
assert s['cmd-release'] == 0
assert s['cmd-use'] == 0
assert s['cmd-watch'] == 0
assert s['cmd-ignore'] == 0
assert s['cmd-bury'] == 0
assert s['cmd-kick'] == 0
assert s['cmd-touch'] == 0
assert s['cmd-stats'] == 1
assert s['cmd-stats-job'] == 0
assert s['cmd-stats-tube'] == 0
assert s['cmd-list-tubes'] == 0
assert s['cmd-list-tube-used'] == 0
assert s['cmd-list-tubes-watched'] == 0
assert s['cmd-pause-tube'] == 0
assert s['job-timeouts'] == 0
assert s['total-jobs'] == 0
assert 'max-job-size' in s
assert s['current-tubes'] == 1
assert s['current-connections'] == 1
assert s['current-producers'] == 0
assert s['current-workers'] == 0
assert s['current-waiting'] == 0
assert s['total-connections'] == 1
assert 'pid' in s
assert 'version' in s
assert 'rusage-utime' in s
assert 'rusage-stime' in s
assert s['uptime'] == 0
assert s['binlog-oldest-index'] == 0
assert s['binlog-current-index'] == 0
assert s['binlog-records-migrated'] == 0
assert s['binlog-records-written'] == 0
assert 'binlog-max-size' in s
assert 'id' in s
assert 'hostname' in s
def wrapper() -> None:
cmd = [BEANSTALKD_PATH]
if isinstance(address, str):
cmd.extend(["-l", "unix:" + address])
else:
host, port = address
cmd.extend(["-l", host, "-p", str(port)])
with subprocess.Popen(cmd) as beanstalkd:
time.sleep(0.1)
try:
with Client(address, use=use, watch=watch) as c:
test(c)
finally:
beanstalkd.terminate()
def test_drain_mode() -> None:
cmd = [BEANSTALKD_PATH, "-l", "unix:" + DEFAULT_UNIX_ADDRESS]
with subprocess.Popen(cmd) as beanstalkd:
time.sleep(0.1)
try:
with Client(address=DEFAULT_UNIX_ADDRESS) as c:
assert c.put(b"first") == 1
beanstalkd.send_signal(signal.SIGUSR1)
time.sleep(0.1)
with pytest.raises(DrainingError):
c.put(b"second")
assert c.stats()["draining"] == "true"
finally:
beanstalkd.terminate()
def wrapper() -> None:
cmd = [BEANSTALKD_PATH]
if isinstance(address, str):
cmd.extend(['-l', 'unix:' + address])
else:
host, port = address
cmd.extend(['-l', host, '-p', str(port)])
with subprocess.Popen(cmd) as beanstalkd:
time.sleep(0.1)
try:
with Client(address, **kwargs) as c:
test(c)
finally:
beanstalkd.terminate()
def test_stats_tube(c: Client) -> None:
assert c.stats_tube('default') == {
'name': 'default',
'current-jobs-urgent': 0,
'current-jobs-ready': 0,
'current-jobs-reserved': 0,
'current-jobs-delayed': 0,
'current-jobs-buried': 0,
'total-jobs': 0,
'current-using': 0,
'current-watching': 1,
'current-waiting': 0,
'cmd-delete': 0,
'cmd-pause-tube': 0,
'pause': 0,
'pause-time-left': 0,
}
def test_stats_tube(c: Client) -> None:
assert c.stats_tube("default") == {
"name": "default",
"current-jobs-urgent": 0,
"current-jobs-ready": 0,
"current-jobs-reserved": 0,
"current-jobs-delayed": 0,
"current-jobs-buried": 0,
"total-jobs": 0,
"current-using": 0,
"current-watching": 1,
"current-waiting": 0,
"cmd-delete": 0,
"cmd-pause-tube": 0,
"pause": 0,
"pause-time-left": 0,
}
def test_delays(c: Client) -> None:
with assert_seconds(1):
c.put(b"delayed", delay=1)
job = c.reserve()
assert job.body == b"delayed"
with assert_seconds(2):
c.release(job, delay=2)
with pytest.raises(TimedOutError):
c.reserve(timeout=1)
job = c.reserve(timeout=1)
c.bury(job)
with pytest.raises(TimedOutError):
c.reserve(timeout=0)
def test_not_ignored(c: Client) -> None:
with pytest.raises(NotIgnoredError):
c.ignore("default")
def test_delete_job_reserved_by_other(c: Client) -> None:
c.put(b"", ttr=1)
with Client(DEFAULT_INET_ADDRESS) as other:
job = other.reserve()
with pytest.raises(NotFoundError):
c.delete(job)
def test_job_not_found(c: Client) -> None:
with pytest.raises(NotFoundError):
c.delete(87)
def test_max_job_size(c: Client) -> None:
with pytest.raises(JobTooBigError):
c.put(bytes(2 ** 16))
def test_pause_tube(c: Client) -> None:
c.put(b"")
with assert_seconds(1):
c.pause_tube("default", 1)
c.reserve()
def test_watching(c: Client) -> None:
assert c.watching() == ["default"]
c.watch("another")
assert set(c.watching()) == {"default", "another"}
