def test_can_count_queues_properly(self):
try:
loaded_procs.clear()
# Put some jobs on the queue
self._add_jobs_to_queue('high', 2)
self._add_jobs_to_queue('bottom', 4)
# Now fake a job being active for one of them
for idx, queue_name in enumerate(['high', 'bottom']):
queue = Queue(queue_name, connection=FakeStrictRedis())
registry = StartedJobRegistry(queue_name, queue.connection)
# Passing in a negative score is important here, otherwise the job will be recognized as expired
registry.connection.zadd(registry.key, -1,
'job_id_{}'.format(idx))
# Load the HF procs
procs = load_procs(*(
'tests.contrib.django.testapp.rq_test_procs.WorkerProc',
'tests.contrib.django.testapp.rq_test_procs.AnotherWorkerProc'
))
# Total should be all queued + 1 active for each
assert sum([proc.quantity()
for proc_name, proc in procs.items()]) == 8
finally:
loaded_procs.clear()
def test_transport_shutdown(sentry_init):
sentry_init(integrations=[RqIntegration()])
events_r, events_w = os.pipe()
events_r = os.fdopen(events_r, "rb", 0)
events_w = os.fdopen(events_w, "wb", 0)
def capture_event(event):
events_w.write(json.dumps(event).encode("utf-8"))
events_w.write(b"\n")
def flush(timeout=None, callback=None):
events_w.write(b"flush\n")
Hub.current.client.transport.capture_event = capture_event
Hub.current.client.flush = flush
queue = rq.Queue(connection=FakeStrictRedis())
worker = rq.Worker([queue], connection=queue.connection)
queue.enqueue(crashing_job, foo=42)
worker.work(burst=True)
event = events_r.readline()
event = json.loads(event.decode("utf-8"))
exception, = event["exception"]["values"]
assert exception["type"] == "ZeroDivisionError"
assert events_r.readline() == b"flush\n"
def test_job_progress_included_in_json(self, uuid4Mock):
"""Tests that after posting to 'wordset_create', the response at 'wordset_create_progress json' contains
attributes indicating the state of the job."""
# mocked Redis server
conn = FakeStrictRedis()
# replace redis connection with conn, replace the queue used by the view with a queue that uses conn
# and immediately runs the task in the current thread
with mock.patch.object(views,
"rq_queue",
new=Queue(is_async=False, connection=conn)):
with mock.patch.object(views, "redis_cursor", new=conn):
self.client.post('/words/wordset/create/', {
'name': 'test1',
'words': 'word\r\ntest'
})
job = Job.fetch(self.job_id, connection=conn)
logger.info(f'job {job}, connection {job.connection}')
logger.info(f'status {job.get_status()}, meta {job.meta}')
response = self.client.get(
reverse("wordset_create_progress json",
args=[self.job_id]))
content = json.loads(response.content)
self.assertIn('status', content)
self.assertIn('potential_words', content)
self.assertIn('processed_words', content)
self.assertIn('recognized_words', content)
def test_retrieve_missing(tmpdir):
source_root = tmpdir.mkdir("source")
repo_root = tmpdir.mkdir("repo")
# Create new source files
(source_root.join("simple_2019-03-01.txt")).open("w").close()
(source_root.join("simple_2019-03-02.txt")).open("w").close()
(source_root.join("simple_2019-03-03.txt")).open("w").close()
(source_root.join("simple_2019-03-04.txt")).open("w").close()
p = Profile("foo", FilesystemDriver, dict(root=str(source_root)))
r = Repository(
"foo",
period="1 days",
start=dt.datetime(2019, 3, 1),
profile=p,
targets=dict(default="empty_{time:%Y-%m-%d}.dat"),
configuration=dict(patterns=dict(
default="simple_{time:%Y-%m-%d}.txt")),
)
ref_time = dt.datetime(2019, 3, 5)
redis_conn = FakeStrictRedis()
runner.retrieve_missing(repo_root, [r],
redis_conn=redis_conn,
is_async=False,
ref_time=ref_time)
assert len(list(glob.glob(str(repo_root.join("foo/*.dat"))))) == 4
def app():
app = Flask(__name__)
redis = FakeStrictRedis()
app.session_interface = RedisSessionInterface(redis)
# pylint: disable=unused-variable
@app.route("/")
def index():
return "Hello world", 200
@app.route("/private")
def private():
if "user_id" not in session:
return "Not OK", 403
return "Hello world", 200
@app.route("/login")
def login():
session["user_id"] = 123
return "OK", 200
@app.route("/logout")
def logout():
session.clear()
return "OK", 200
with app.app_context():
yield app
def setup(self):
self.app.conf.add_defaults({
'REDBEAT_KEY_PREFIX': 'rb-tests:',
'redbeat_key_prefix': 'rb-tests:',
})
self.app.redbeat_redis = FakeStrictRedis(decode_responses=True)
self.app.redbeat_redis.flushdb()
def init_app(self, app, **kwargs):
redis_url = app.config.setdefault("REDIS_URL", "redis://localhost:6379/0")
if self._redis_client is None:
if redis_url == ":fake:":
self._redis_client = FakeStrictRedis()
else:
self._redis_client = Redis.from_url(redis_url, **kwargs)
def test_basic(sentry_init, capture_events):
sentry_init(integrations=[RqIntegration()])
events = capture_events()
queue = rq.Queue(connection=FakeStrictRedis())
worker = rq.SimpleWorker([queue], connection=queue.connection)
queue.enqueue(crashing_job, foo=42)
worker.work(burst=True)
(event, ) = events
(exception, ) = event["exception"]["values"]
assert exception["type"] == "ZeroDivisionError"
assert exception["mechanism"]["type"] == "rq"
assert exception["stacktrace"]["frames"][-1]["vars"]["foo"] == "42"
assert event["transaction"] == "tests.integrations.rq.test_rq.crashing_job"
assert event["extra"]["rq-job"] == {
"args": [],
"description": "tests.integrations.rq.test_rq.crashing_job(foo=42)",
"func": "tests.integrations.rq.test_rq.crashing_job",
"job_id": event["extra"]["rq-job"]["job_id"],
"kwargs": {
"foo": 42
},
}
def test_coverage_summary_by_changeset(coverage_builds):
from rq import Queue
from codecoverage_backend import api
from tests.conftest import mock_coverage_by_changeset_job_success
# patch the queue to be sync to allow it run without workers. http://python-rq.org/docs/testing/
with mock.patch('codecoverage_backend.api.q',
Queue(connection=FakeStrictRedis())) as q:
# patch the mock_coverage_by_changeset
with mock.patch('codecoverage_backend.api.coverage_by_changeset_job',
mock_coverage_by_changeset_job_success):
# Get changeset coverage information
for changeset, expected in coverage_builds['summary'].items():
result, code = api.coverage_summary_by_changeset(changeset)
assert code == 202
# test that in the case of exception it will return 500
result, code = api.coverage_summary_by_changeset(
'mozilla test changeset')
assert code == 202
# run simple worker to run all tasks
w = SimpleWorker([q], connection=q.connection)
w.work(burst=True)
# Everything should be 200 now
for changeset, expected in coverage_builds['summary'].items():
result, code = api.coverage_summary_by_changeset(changeset)
assert result == expected
assert code == 200
# except the incorrect changeset, should be 500
result, code = api.coverage_summary_by_changeset(
'mozilla test changeset')
assert code == 500
def get_opts(use_redis=False):
if use_redis:
return {
"rc": FakeStrictRedis(decode_responses=True),
"use_in_memory_on_failure": False,
}
return {"use_in_memory_on_failure": True}
def mq():
"""
A mock queue that processes the job in the same thread via a mocked
Redis, so that the application flow can be simulated in side-effect-free tests.
"""
q = Queue(is_async=False, connection=FakeStrictRedis())
yield q
def test_redis_load(self):
redis = RedisHelper(redis_client=FakeStrictRedis())
user_profile = dict(user_id=123456789, action='test', content=dict(test='test'))
redis.save(user_profile)
result = redis.load(user_profile)
assert result == user_profile
def redis():
r = FakeStrictRedis()
r.sadd("packages-snapshot", "test1", "test2", "test3")
r.hmset("profiles-snapshot", {"8devices_carambola": "ramips/rt305x"})
r.sadd("targets-snapshot", "testtarget/testsubtarget")
r.hmset("profiles-snapshot", {"testprofile": "testtarget/testsubtarget"})
yield r
def scheduler(config: Dict[str, Any]) -> ChaosPlatformScheduler:
queue = create_scheduler_queue("myqueue",
FakeStrictRedis(singleton=False),
is_async=False)
sched = create_scheduler(queue, config)
sched.interval = 1
return sched
def test_transaction_no_error(
sentry_init,
capture_events,
DictionaryContaining # noqa:N803
):
sentry_init(integrations=[RqIntegration()], traces_sample_rate=1.0)
events = capture_events()
queue = rq.Queue(connection=FakeStrictRedis())
worker = rq.SimpleWorker([queue], connection=queue.connection)
queue.enqueue(do_trick, "Maisey", trick="kangaroo")
worker.work(burst=True)
envelope = events[0]
assert envelope["type"] == "transaction"
assert envelope["contexts"]["trace"]["op"] == "rq.task"
assert envelope["transaction"] == "tests.integrations.rq.test_rq.do_trick"
assert envelope["extra"]["rq-job"] == DictionaryContaining({
"args": ["Maisey"],
"kwargs": {
"trick": "kangaroo"
},
"func":
"tests.integrations.rq.test_rq.do_trick",
"description":
"tests.integrations.rq.test_rq.do_trick('Maisey', trick='kangaroo')",
})
def test_traces_sampler_gets_correct_values_in_sampling_context(
sentry_init,
DictionaryContaining,
ObjectDescribedBy # noqa:N803
):
traces_sampler = mock.Mock(return_value=True)
sentry_init(integrations=[RqIntegration()], traces_sampler=traces_sampler)
queue = rq.Queue(connection=FakeStrictRedis())
worker = rq.SimpleWorker([queue], connection=queue.connection)
queue.enqueue(do_trick, "Bodhi", trick="roll over")
worker.work(burst=True)
traces_sampler.assert_any_call(
DictionaryContaining({
"rq_job":
ObjectDescribedBy(
type=rq.job.Job,
attrs={
"description":
"tests.integrations.rq.test_rq.do_trick('Bodhi', trick='roll over')",
"result": "Bodhi, can you roll over? Good dog!",
"func_name": "tests.integrations.rq.test_rq.do_trick",
"args": ("Bodhi", ),
"kwargs": {
"trick": "roll over"
},
},
),
}))
def before(self):
super(TestEnd, self).before()
self.redis = FakeStrictRedis()
self.redis.zadd('game:1:scores', 100, 100)
self.redis.zadd('game:1:scores', 110, 110)
self.resource = End(self.redis)
self.api.add_route('/end/{game_id}', self.resource)
self.api.req_options.auto_parse_form_urlencoded = True
def run(i):
keys = gen_items("keys-%04d" % (i, ), 501)
vals = gen_items("vals-%04d" % (i, ), 60)
rc = StrictRedis(host=host, port=port)
fake = FakeStrictRedis()
cmds_list = list(gen_cmds(fake, rc, keys, vals))
check(cmds_list)
delete_all(rc, keys)
def setUp(self):
self.registry = CollectorRegistry()
# Mock time so _created values are fixed.
self.old_time = time.time
time.time = lambda: 123.456
redis_app = FakeStrictRedis()
self.redis_provider = get_redis_provider(redis_app)
def test_redis(self):
redis = RedisController(redis_client=FakeStrictRedis())
user_profile = dict(test='test')
assert redis.save(USER_ID, ACTION, user_profile)
assert user_profile == redis.load(USER_ID, ACTION)
# Delete
assert redis.clean(USER_ID, ACTION)
def redis():
fake_redis = FakeStrictRedis()
with patch("autotester.cli.redis_connection", return_value=fake_redis):
with patch(
"autotester.server.utils.redis_management.redis_connection",
return_value=fake_redis,
):
yield fake_redis
def redis():
r = FakeStrictRedis()
r.sadd("packages-snapshot-testtarget/testsubtarget", "test1", "test2",
"test3")
r.hmset("profiles-snapshot", {"testprofile": "testtarget/testsubtarget"})
r.hmset("mapping-snapshot", {"testvendor,testprofile": "testprofile"})
r.sadd("targets-snapshot", "testtarget/testsubtarget")
yield r
def setUp(self):
server = FakeServer()
server.connected = True
self.store = RedisStore(prefix='prefix:',
redis_class=FakeStrictRedis,
server=server)
self.redis = FakeStrictRedis(server=server)
super(RedisStoreTestCase, self).setUp()
def setup_class(cls):
super(TestSingleUseLock, cls).setup_class()
os.environ['TEST_SESSION_LOCK_INTERVAL'] = '3'
os.environ['LOCKS_AUTH'] = 'ukwa-admin:testpass'
cls.testapp = cls.get_test_app()
cls.redis = FakeStrictRedis(decode_responses=True)
def full_rig_without_s3():
fake_redis_connection = FakeStrictRedis()
queue = Queue(async=False, connection=fake_redis_connection)
with patch('backend.apis.upload.notify_matcher', return_value=None):
with patch('backend.apis.upload.get_redis_connection', return_value=fake_redis_connection):
with patch('backend.apis.upload.get_q', return_value=queue):
with patch.dict('backend.utils.app_config', SAMPLE_CONFIG):
with rig_test_client() as (app, engine):
authenticate(app)
yield app, engine
def run_job():
queue = Queue(connection=FakeStrictRedis())
worker = SimpleWorker([queue], connection=queue.connection)
def inner(fn, *a, **kw):
job = queue.enqueue(fn, *a, **kw)
worker.work(burst=True)
return job
return inner
def post():
if app_flask.config['TESTING']:
services = [ServiceTypes.LOG]
q = rq.Queue(name='test', is_async=False, connection=FakeStrictRedis())
else:
services = ENABLED_SERIVCES
q = queue
for service_type in services:
job = q.enqueue(task_handler, {'service_type': service_type, 'message': request.get_json()['message']})
return request.get_json()['message'], status.HTTP_200_OK
def test_create_scheduler(config: Dict[str, Any]):
queue = create_scheduler_queue("myqueue",
FakeStrictRedis(),
is_async=False)
sched = create_scheduler(queue, config)
assert sched is not None
job = queue.enqueue(count_words, "hello world")
assert job.is_finished
assert job.result == 2
def _init_cookie_tracker(self, redis=None):
"""Initialize the CookieTracker
:param redis: Optional redis instance to be used
Defaults to FakeStrictRedis
:return: The initialized cookie tracker
:rtype: CookieTracker
"""
if redis is None:
redis = FakeStrictRedis()
return CookieTracker(redis)
def test_job_with_retries(sentry_init, capture_events):
sentry_init(integrations=[RqIntegration()])
events = capture_events()
queue = rq.Queue(connection=FakeStrictRedis())
worker = rq.SimpleWorker([queue], connection=queue.connection)
queue.enqueue(crashing_job, foo=42, retry=rq.Retry(max=1))
worker.work(burst=True)
assert len(events) == 1
