def test_iter_native(self):
backend = SimpleBackend()
results = [self.app.AsyncResult(uuid(), backend=backend) for i in range(10)]
ts = self.app.GroupResult(uuid(), results)
ts.app.backend = backend
backend.ids = [result.id for result in results]
self.assertEqual(len(list(ts.iter_native())), 10)
def test_with_parent(self):
x = self.app.AsyncResult(uuid())
x.parent = self.app.AsyncResult(uuid())
y = result_from_tuple(x.as_tuple(), self.app)
assert y == x
assert y.parent == x.parent
assert isinstance(y.parent, AsyncResult)
def test_add_to_chord(self):
self.app.backend = Mock(name='backend')
sig = self.add.s(2, 2)
sig.delay = Mock(name='sig.delay')
self.adds.request.group = uuid()
self.adds.request.id = uuid()
with self.assertRaises(ValueError):
# task not part of chord
self.adds.run(sig)
self.adds.request.chord = self.add.s()
res1 = self.adds.run(sig, True)
self.assertEqual(res1, sig)
self.assertTrue(sig.options['task_id'])
self.assertEqual(sig.options['group_id'], self.adds.request.group)
self.assertEqual(sig.options['chord'], self.adds.request.chord)
self.assertFalse(sig.delay.called)
self.app.backend.add_to_chord.assert_called_with(
self.adds.request.group, sig.freeze(),
)
self.app.backend.reset_mock()
sig2 = self.add.s(4, 4)
sig2.delay = Mock(name='sig2.delay')
res2 = self.adds.run(sig2)
self.assertEqual(res2, sig2.delay.return_value)
self.assertTrue(sig2.options['task_id'])
self.assertEqual(sig2.options['group_id'], self.adds.request.group)
self.assertEqual(sig2.options['chord'], self.adds.request.chord)
sig2.delay.assert_called_with()
self.app.backend.add_to_chord.assert_called_with(
self.adds.request.group, sig2.freeze(),
)
def test_iterate_raises(self):
ar = MockAsyncResultFailure(uuid(), app=self.app)
ts = self.app.GroupResult(uuid(), [ar])
with pytest.warns(CPendingDeprecationWarning):
it = ts.iterate()
with pytest.raises(KeyError):
next(it)
def test_restore_group(self, mock_get_database):
self.backend.groupmeta_collection = MONGODB_GROUP_COLLECTION
mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
mock_collection = Mock()
mock_collection.find_one.return_value = {
'_id': sentinel.taskset_id,
'result': [uuid(), uuid()],
'date_done': 1,
}
self.backend.decode.side_effect = lambda r: r
mock_get_database.return_value = mock_database
mock_database.__getitem__.return_value = mock_collection
ret_val = self.backend._restore_group(sentinel.taskset_id)
mock_get_database.assert_called_once_with()
mock_collection.find_one.assert_called_once_with(
{'_id': sentinel.taskset_id})
self.assertItemsEqual(
['date_done', 'result', 'task_id'],
list(ret_val.keys()),
)
mock_collection.find_one.return_value = None
self.backend._restore_group(sentinel.taskset_id)
def test_on_chord_part_return(self, restore):
tb = CacheBackend(backend='memory://', app=self.app)
deps = Mock()
deps.__len__ = Mock()
deps.__len__.return_value = 2
restore.return_value = deps
task = Mock()
task.name = 'foobarbaz'
self.app.tasks['foobarbaz'] = task
task.request.chord = signature(task)
result = self.app.GroupResult(
uuid(),
[self.app.AsyncResult(uuid()) for _ in range(3)],
)
task.request.group = result.id
tb.apply_chord(result, None)
deps.join_native.assert_not_called()
tb.on_chord_part_return(task.request, 'SUCCESS', 10)
deps.join_native.assert_not_called()
tb.on_chord_part_return(task.request, 'SUCCESS', 10)
deps.join_native.assert_called_with(propagate=True, timeout=3.0)
deps.delete.assert_called_with()
def test_poll_result(self):
with self._result_context() as (results, backend, Message):
tid = uuid()
# FFWD's to the latest state.
state_messages = [
Message(task_id=tid, status=states.RECEIVED, seq=1),
Message(task_id=tid, status=states.STARTED, seq=2),
Message(task_id=tid, status=states.FAILURE, seq=3),
]
for state_message in state_messages:
results.put(state_message)
r1 = backend.get_task_meta(tid)
self.assertDictContainsSubset(
{'status': states.FAILURE, 'seq': 3}, r1,
'FFWDs to the last state',
)
# Caches last known state.
tid = uuid()
results.put(Message(task_id=tid))
backend.get_task_meta(tid)
self.assertIn(tid, backend._cache, 'Caches last known state')
self.assertTrue(state_messages[-1].requeued)
# Returns cache if no new states.
results.queue.clear()
assert not results.qsize()
backend._cache[tid] = 'hello'
self.assertEqual(
backend.get_task_meta(tid), 'hello',
'Returns cache if no new states',
)
def test_poll_result(self):
with self._result_context() as (results, backend, Message):
tid = uuid()
# FFWD's to the latest state.
state_messages = [
Message(task_id=tid, status=states.RECEIVED, seq=1),
Message(task_id=tid, status=states.STARTED, seq=2),
Message(task_id=tid, status=states.FAILURE, seq=3),
]
for state_message in state_messages:
results.put(state_message)
r1 = backend.get_task_meta(tid)
# FFWDs to the last state.
assert r1['status'] == states.FAILURE
assert r1['seq'] == 3
# Caches last known state.
tid = uuid()
results.put(Message(task_id=tid))
backend.get_task_meta(tid)
assert tid, backend._cache in 'Caches last known state'
assert state_messages[-1].requeued
# Returns cache if no new states.
results.queue.clear()
assert not results.qsize()
backend._cache[tid] = 'hello'
# returns cache if no new states.
assert backend.get_task_meta(tid) == 'hello'
def test_restore_current_app_fallback(self):
subs = [MockAsyncResultSuccess(uuid(), app=self.app)]
ts = self.app.GroupResult(uuid(), subs)
ts.save()
with pytest.raises(RuntimeError,
message="Test depends on current_app"):
GroupResult.restore(ts.id)
def setup(self):
self.simple_message = self.app.amqp.as_task_v2(
uuid(), 'foo', create_sent_event=True,
)
self.simple_message_no_sent_event = self.app.amqp.as_task_v2(
uuid(), 'foo', create_sent_event=False,
)
def test_with_parent(self):
x = self.app.AsyncResult(uuid())
x.parent = self.app.AsyncResult(uuid())
y = result_from_tuple(x.as_tuple(), self.app)
self.assertEqual(y, x)
self.assertEqual(y.parent, x.parent)
self.assertIsInstance(y.parent, AsyncResult)
def test_iterate_raises(self):
ar = MockAsyncResultFailure(uuid(), app=self.app)
ts = self.app.GroupResult(uuid(), [ar])
with self.assertPendingDeprecation():
it = ts.iterate()
with self.assertRaises(KeyError):
next(it)
def test_iterate_respects_subpolling_interval(self):
r1 = self.app.AsyncResult(uuid())
r2 = self.app.AsyncResult(uuid())
backend = r1.backend = r2.backend = Mock()
backend.subpolling_interval = 10
ready = r1.ready = r2.ready = Mock()
def se(*args, **kwargs):
ready.side_effect = KeyError()
return False
ready.return_value = False
ready.side_effect = se
x = self.app.ResultSet([r1, r2])
with self.dummy_copy():
with patch('celery.result.time') as _time:
with pytest.warns(CPendingDeprecationWarning):
with pytest.raises(KeyError):
list(x.iterate())
_time.sleep.assert_called_with(10)
backend.subpolling_interval = 0
with patch('celery.result.time') as _time:
with pytest.warns(CPendingDeprecationWarning):
with pytest.raises(KeyError):
ready.return_value = False
ready.side_effect = se
list(x.iterate())
_time.sleep.assert_not_called()
def test_add_to_chord(self):
self.app.backend = Mock(name='backend')
sig = self.add.s(2, 2)
sig.delay = Mock(name='sig.delay')
self.adds.request.group = uuid()
self.adds.request.id = uuid()
with pytest.raises(ValueError):
# task not part of chord
self.adds.run(sig)
self.adds.request.chord = self.add.s()
res1 = self.adds.run(sig, True)
assert res1 == sig
assert sig.options['task_id']
assert sig.options['group_id'] == self.adds.request.group
assert sig.options['chord'] == self.adds.request.chord
sig.delay.assert_not_called()
self.app.backend.add_to_chord.assert_called_with(
self.adds.request.group, sig.freeze(),
)
self.app.backend.reset_mock()
sig2 = self.add.s(4, 4)
sig2.delay = Mock(name='sig2.delay')
res2 = self.adds.run(sig2)
assert res2 == sig2.delay.return_value
assert sig2.options['task_id']
assert sig2.options['group_id'] == self.adds.request.group
assert sig2.options['chord'] == self.adds.request.chord
sig2.delay.assert_called_with()
self.app.backend.add_to_chord.assert_called_with(
self.adds.request.group, sig2.freeze(),
)
def test_forget(self):
subs = [MockAsyncResultSuccess(uuid(), app=self.app),
MockAsyncResultSuccess(uuid(), app=self.app)]
ts = self.app.GroupResult(uuid(), subs)
ts.forget()
for sub in subs:
assert sub.forgotten
def test_revoke_from_resultset(self):
self.app.control.revoke = Mock(name="revoke")
uuids = [uuid() for _ in range(10)]
r = self.app.GroupResult(uuid(), [self.app.AsyncResult(x) for x in uuids])
r.revoke()
self.app.control.revoke.assert_called_with(
uuids, connection=None, reply=False, signal=None, terminate=False, timeout=None
)
def test_join_native_raises(self):
ts = self.app.GroupResult(uuid(), [self.app.AsyncResult(uuid())])
ts.iter_native = Mock()
ts.iter_native.return_value = iter([
(uuid(), {'status': states.FAILURE, 'result': KeyError()})
])
with pytest.raises(KeyError):
ts.join_native(propagate=True)
def test_iterate_yields(self):
ar = MockAsyncResultSuccess(uuid(), app=self.app)
ar2 = MockAsyncResultSuccess(uuid(), app=self.app)
ts = self.app.GroupResult(uuid(), [ar, ar2])
with pytest.warns(CPendingDeprecationWarning):
it = ts.iterate()
assert next(it) == 42
assert next(it) == 42
def test_iterate_eager(self):
ar1 = EagerResult(uuid(), 42, states.SUCCESS)
ar2 = EagerResult(uuid(), 42, states.SUCCESS)
ts = self.app.GroupResult(uuid(), [ar1, ar2])
with pytest.warns(CPendingDeprecationWarning):
it = ts.iterate()
assert next(it) == 42
assert next(it) == 42
def test_iterate_eager(self):
ar1 = EagerResult(uuid(), 42, states.SUCCESS)
ar2 = EagerResult(uuid(), 42, states.SUCCESS)
ts = self.app.GroupResult(uuid(), [ar1, ar2])
with self.assertPendingDeprecation():
it = ts.iterate()
self.assertEqual(next(it), 42)
self.assertEqual(next(it), 42)
def test_iterate_yields(self):
ar = MockAsyncResultSuccess(uuid(), app=self.app)
ar2 = MockAsyncResultSuccess(uuid(), app=self.app)
ts = self.app.GroupResult(uuid(), [ar, ar2])
with self.assertPendingDeprecation():
it = ts.iterate()
self.assertEqual(next(it), 42)
self.assertEqual(next(it), 42)
def test_apply_chord(self):
tb = CacheBackend(backend='memory://', app=self.app)
result = self.app.GroupResult(
uuid(),
[self.app.AsyncResult(uuid()) for _ in range(3)],
)
tb.apply_chord(result, None)
assert self.app.GroupResult.restore(result.id, backend=tb) == result
def test_GroupResult_with_parent(self):
parent = self.app.AsyncResult(uuid())
result = self.app.GroupResult(
uuid(), [self.app.AsyncResult(uuid()) for _ in range(10)],
parent
)
second_result = result_from_tuple(result.as_tuple(), self.app)
assert second_result == result
assert second_result.parent == parent
def test_eq_ne(self):
ts = self.app.GroupResult(uuid(), [self.app.AsyncResult(uuid())])
ts2 = self.app.GroupResult(ts.id, ts.results)
ts3 = self.app.GroupResult(uuid(), [self.app.AsyncResult(uuid())])
ts4 = self.app.GroupResult(ts.id, [self.app.AsyncResult(uuid())])
assert ts == ts2
assert ts != ts3
assert ts != ts4
assert ts != object()
def test_eq_ne(self):
ts = self.app.GroupResult(uuid(), [self.app.AsyncResult(uuid())])
ts2 = self.app.GroupResult(ts.id, ts.results)
ts3 = self.app.GroupResult(uuid(), [self.app.AsyncResult(uuid())])
ts4 = self.app.GroupResult(ts.id, [self.app.AsyncResult(uuid())])
self.assertEqual(ts, ts2)
self.assertNotEqual(ts, ts3)
self.assertNotEqual(ts, ts4)
self.assertNotEqual(ts, object())
def test_get_many(self):
data = {uuid(): 'foo', uuid(): 'bar', uuid(): 'baz'}
tb = FilesystemBackend(app=self.app, url=self.url)
for key, value in data.items():
tb.mark_as_done(key, value)
for key, result in tb.get_many(data.keys()):
assert result['result'] == data[key]
def test_save_restore_delete_group(self):
tid = uuid()
tsr = self.app.GroupResult(
tid, [self.app.AsyncResult(uuid()) for _ in range(10)],
)
self.b.save_group(tid, tsr)
self.b.restore_group(tid)
assert self.b.restore_group(tid) == tsr
self.b.delete_group(tid)
assert self.b.restore_group(tid) is None
def test_join_native(self):
backend = SimpleBackend()
results = [self.app.AsyncResult(uuid(), backend=backend) for i in range(10)]
ts = self.app.GroupResult(uuid(), results)
ts.app.backend = backend
backend.ids = [result.id for result in results]
res = ts.join_native()
self.assertEqual(res, list(range(10)))
callback = Mock(name="callback")
self.assertFalse(ts.join_native(callback=callback))
callback.assert_has_calls([call(r.id, i) for i, r in enumerate(ts.results)])
def test_store_result_parent_id(self):
tid = uuid()
pid = uuid()
state = 'SUCCESS'
result = 10
request = Context(parent_id=pid)
self.b.store_result(
tid, state=state, result=result, request=request,
)
stored_meta = self.b.decode(self.b.get(self.b.get_key_for_task(tid)))
assert stored_meta['parent_id'] == request.parent_id
def test_save_restore_delete_group(self):
group_id = uuid()
result_ids = [uuid() for i in range(10)]
results = list(map(result.AsyncResult, result_ids))
res = result.GroupResult(group_id, results)
res.save(backend=self.b)
saved = result.GroupResult.restore(group_id, backend=self.b)
assert saved.results == results
assert saved.id == group_id
saved.delete(backend=self.b)
assert result.GroupResult.restore(group_id, backend=self.b) is None
def test_mark_as_done_writes_file(self):
tb = FilesystemBackend(app=self.app, url=self.url)
tb.mark_as_done(uuid(), 42)
assert len(os.listdir(self.directory)) == 1
def test_done_task_is_SUCCESS(self):
tb = FilesystemBackend(app=self.app, url=self.url)
tid = uuid()
tb.mark_as_done(tid, 42)
assert tb.get_state(tid) == states.SUCCESS
def test_forget_deletes_file(self):
tb = FilesystemBackend(app=self.app, url=self.url)
tid = uuid()
tb.mark_as_done(tid, 42)
tb.forget(tid)
assert len(os.listdir(self.directory)) == 0
import pytest
import datetime
from pickle import loads, dumps
from case import ANY, MagicMock, Mock, mock, patch, sentinel, skip
from kombu.exceptions import EncodeError
from celery import uuid
from celery import states
from celery.backends.mongodb import InvalidDocument, MongoBackend
from celery.exceptions import ImproperlyConfigured
COLLECTION = 'taskmeta_celery'
TASK_ID = uuid()
MONGODB_HOST = 'localhost'
MONGODB_PORT = 27017
MONGODB_USER = '******'
MONGODB_PASSWORD = '******'
MONGODB_DATABASE = 'testing'
MONGODB_COLLECTION = 'collection1'
MONGODB_GROUP_COLLECTION = 'group_collection1'
@skip.unless_module('pymongo')
class test_MongoBackend:
default_url = 'mongodb://*****:*****@hostname.dom/database'
replica_set_url = (
'mongodb://*****:*****@hostname.dom,'
def longtask():
task_id = uuid() # create your task_id
task = long_task.apply_async(task_id=task_id)
return jsonify({}), 202, {'Location': url_for('taskstatus', task_id=task.id)}
def setup(self):
self.size = 10
self.ts = self.app.GroupResult(
uuid(),
make_mock_group(self.app, self.size),
)
def test_AsyncResult(self):
task_id = uuid()
result = self.retry_task.AsyncResult(task_id)
assert result.backend == self.retry_task.backend
assert result.id == task_id
def test_apply_chord(self):
tb = CacheBackend(backend='memory://', app=self.app)
gid, res = uuid(), [self.app.AsyncResult(uuid()) for _ in range(3)]
tb.apply_chord(group(app=self.app), (), gid, {}, result=res)
def test_add_to_chord(self):
b = self.Backend('redis://', app=self.app)
gid = uuid()
b.add_to_chord(gid, 'sig')
b.client.incr.assert_called_with(b.get_key_for_group(gid, '.t'), 1)
def test_mark_as_started(self):
tb = DatabaseBackend(self.uri, app=self.app)
tid = uuid()
tb.mark_as_started(tid)
assert tb.get_state(tid) == states.STARTED
def test_mark_as_revoked(self):
tb = DatabaseBackend(self.uri, app=self.app)
tid = uuid()
tb.mark_as_revoked(tid)
assert tb.get_state(tid) == states.REVOKED
def resubmit_jobs(context):
"""
logic to resubmit the job
:param context: contents from _context.json
"""
# iterate through job ids and query to get the job json
increment_by = None
new_priority = None
if "job_priority_increment" in context:
increment_by = context["job_priority_increment"]
else:
new_priority = context["new_job_priority"]
retry_count_max = context['retry_count_max']
if isinstance(context['retry_job_id'], list):
retry_job_ids = context['retry_job_id']
else:
retry_job_ids = [context['retry_job_id']]
for job_id in retry_job_ids:
print(("Validating retry job: {}".format(job_id)))
try:
doc = query_es(job_id)
if doc['hits']['total']['value'] == 0:
print('job id %s not found in Elasticsearch. Continuing.' %
job_id)
continue
doc = doc["hits"]["hits"][0]
job_json = doc["_source"]["job"]
task_id = doc["_source"]["uuid"]
index = doc["_index"]
_id = doc["_id"]
# don't retry a retry
if job_json['type'].startswith('job-lw-mozart-retry'):
print("Cannot retry retry job %s. Skipping" % job_id)
continue
# check retry_remaining_count
if 'retry_count' in job_json:
if job_json['retry_count'] < retry_count_max:
job_json['retry_count'] = int(job_json['retry_count']) + 1
else:
print(
"For job {}, retry_count now is {}, retry_count_max limit of {} reached. Cannot retry again."
.format(job_id, job_json['retry_count'],
retry_count_max))
continue
else:
job_json['retry_count'] = 1
job_json["job_info"]["dedup"] = False
# clean up job execution info
for i in ('duration', 'execute_node', 'facts', 'job_dir',
'job_url', 'metrics', 'pid', 'public_ip', 'status',
'stderr', 'stdout', 'time_end', 'time_queued',
'time_start'):
if i in job_json.get('job_info', {}):
del job_json['job_info'][i]
# set queue time
job_json['job_info']['time_queued'] = datetime.utcnow().isoformat(
) + 'Z'
# reset priority
old_priority = job_json['priority']
job_json['priority'] = get_new_job_priority(
old_priority=old_priority,
increment_by=increment_by,
new_priority=new_priority)
# get state
task = app.AsyncResult(task_id)
state = task.state
# revoke
job_id = job_json['job_id']
try:
revoke(task_id, state)
print("revoked original job: %s (%s)" % (job_id, task_id))
except:
print("Got error issuing revoke on job %s (%s): %s" %
(job_id, task_id, traceback.format_exc()))
print("Continuing.")
# generate celery task id
new_task_id = uuid()
job_json['task_id'] = new_task_id
# delete old job status
delete_by_id(index, _id)
# log queued status
job_status_json = {
'uuid':
new_task_id,
'job_id':
job_id,
'payload_id':
job_json['job_info']['job_payload']['payload_task_id'],
'status':
'job-queued',
'job':
job_json
}
log_job_status(job_status_json)
# submit job
queue = job_json['job_info']['job_queue']
run_job.apply_async(
(job_json, ),
queue=queue,
time_limit=job_json['job_info']['time_limit'],
soft_time_limit=job_json['job_info']['soft_time_limit'],
priority=job_json['priority'],
task_id=new_task_id)
except Exception as ex:
print("[ERROR] Exception occurred {0}:{1} {2}".format(
type(ex), ex, traceback.format_exc()),
file=sys.stderr)
def setup(self):
self.app.conf.result_serializer = 'pickle'
self.tb = CacheBackend(backend='memory://', app=self.app)
self.tid = uuid()
self.old_get_best_memcached = backends['memcache']
backends['memcache'] = lambda: (DummyClient, ensure_bytes)
def create_task_result(self):
id = uuid()
taskmeta, created = TaskResult.objects.get_or_create(task_id=id)
return taskmeta
def test_get_many_times_out(self):
tasks = [uuid() for _ in range(4)]
self.b._cache[tasks[1]] = {'status': 'PENDING'}
with pytest.raises(self.b.TimeoutError):
list(self.b.get_many(tasks, timeout=0.01, interval=0.01))
def start_analyzers(
analyzers_to_execute,
analyzers_config,
runtime_configuration,
job_id,
md5,
is_sample,
):
set_job_status(job_id, "running")
if is_sample:
file_path, filename = get_filepath_filename(job_id)
else:
observable_name, observable_classification = get_observable_data(
job_id)
for analyzer in analyzers_to_execute:
ac = analyzers_config[analyzer]
try:
module = ac.get("python_module", None)
if not module:
raise AnalyzerConfigurationException(
f"no python_module available in config for {analyzer} analyzer?!"
)
additional_config_params = ac.get("additional_config_params", {})
adjust_analyzer_config(runtime_configuration,
additional_config_params, analyzer)
# get celery queue
queue = ac.get("queue", "default")
if queue not in CELERY_QUEUES:
logger.error(
f"Analyzer {analyzers_to_execute} has a wrong queue."
f" Setting to default")
queue = "default"
# construct arguments
if is_sample:
# check if we should run the hash instead of the binary
run_hash = ac.get("run_hash", False)
if run_hash:
# check which kind of hash the analyzer needs
run_hash_type = ac.get("run_hash_type", "md5")
if run_hash_type == "md5":
hash_value = md5
elif run_hash_type == "sha256":
hash_value = generate_sha256(job_id)
else:
error_message = (
f"only md5 and sha256 are supported "
f"but you asked {run_hash_type}. job_id: {job_id}")
raise AnalyzerConfigurationException(error_message)
# run the analyzer with the hash
args = [
f"observable_analyzers.{module}",
analyzer,
job_id,
hash_value,
"hash",
additional_config_params,
]
else:
# run the analyzer with the binary
args = [
f"file_analyzers.{module}",
analyzer,
job_id,
file_path,
filename,
md5,
additional_config_params,
]
else:
# observables analyzer case
args = [
f"observable_analyzers.{module}",
analyzer,
job_id,
observable_name,
observable_classification,
additional_config_params,
]
# run analyzer with a celery task asynchronously
stl = ac.get("soft_time_limit", 300)
t_id = uuid()
celery_app.send_task(
"run_analyzer",
args=args,
queue=queue,
soft_time_limit=stl,
task_id=t_id,
)
# to track task_id by job_id
task_ids = cache.get(job_id)
if isinstance(task_ids, list):
task_ids.append(t_id)
else:
task_ids = [t_id]
cache.set(job_id, task_ids)
except (AnalyzerConfigurationException, AnalyzerRunException) as e:
err_msg = f"({analyzer}, job_id #{job_id}) -> Error: {e}"
logger.error(err_msg)
set_failed_analyzer(analyzer, job_id, err_msg)
def create_group_result(self):
id = uuid()
taskmeta, created = GroupResult.objects.get_or_create(group_id=id)
return taskmeta