Ejemplo n.º 1
0
 def test_iter_native(self):
     backend = SimpleBackend()
     results = [self.app.AsyncResult(uuid(), backend=backend) for i in range(10)]
     ts = self.app.GroupResult(uuid(), results)
     ts.app.backend = backend
     backend.ids = [result.id for result in results]
     self.assertEqual(len(list(ts.iter_native())), 10)
Ejemplo n.º 2
0
 def test_with_parent(self):
     x = self.app.AsyncResult(uuid())
     x.parent = self.app.AsyncResult(uuid())
     y = result_from_tuple(x.as_tuple(), self.app)
     assert y == x
     assert y.parent == x.parent
     assert isinstance(y.parent, AsyncResult)
Ejemplo n.º 3
0
    def test_add_to_chord(self):
        self.app.backend = Mock(name='backend')

        sig = self.add.s(2, 2)
        sig.delay = Mock(name='sig.delay')
        self.adds.request.group = uuid()
        self.adds.request.id = uuid()

        with self.assertRaises(ValueError):
            # task not part of chord
            self.adds.run(sig)
        self.adds.request.chord = self.add.s()

        res1 = self.adds.run(sig, True)
        self.assertEqual(res1, sig)
        self.assertTrue(sig.options['task_id'])
        self.assertEqual(sig.options['group_id'], self.adds.request.group)
        self.assertEqual(sig.options['chord'], self.adds.request.chord)
        self.assertFalse(sig.delay.called)
        self.app.backend.add_to_chord.assert_called_with(
            self.adds.request.group, sig.freeze(),
        )

        self.app.backend.reset_mock()
        sig2 = self.add.s(4, 4)
        sig2.delay = Mock(name='sig2.delay')
        res2 = self.adds.run(sig2)
        self.assertEqual(res2, sig2.delay.return_value)
        self.assertTrue(sig2.options['task_id'])
        self.assertEqual(sig2.options['group_id'], self.adds.request.group)
        self.assertEqual(sig2.options['chord'], self.adds.request.chord)
        sig2.delay.assert_called_with()
        self.app.backend.add_to_chord.assert_called_with(
            self.adds.request.group, sig2.freeze(),
        )
Ejemplo n.º 4
0
 def test_iterate_raises(self):
     ar = MockAsyncResultFailure(uuid(), app=self.app)
     ts = self.app.GroupResult(uuid(), [ar])
     with pytest.warns(CPendingDeprecationWarning):
         it = ts.iterate()
     with pytest.raises(KeyError):
         next(it)
Ejemplo n.º 5
0
    def test_restore_group(self, mock_get_database):
        self.backend.groupmeta_collection = MONGODB_GROUP_COLLECTION

        mock_database = MagicMock(spec=['__getitem__', '__setitem__'])
        mock_collection = Mock()
        mock_collection.find_one.return_value = {
            '_id': sentinel.taskset_id,
            'result': [uuid(), uuid()],
            'date_done': 1,
        }
        self.backend.decode.side_effect = lambda r: r

        mock_get_database.return_value = mock_database
        mock_database.__getitem__.return_value = mock_collection

        ret_val = self.backend._restore_group(sentinel.taskset_id)

        mock_get_database.assert_called_once_with()
        mock_collection.find_one.assert_called_once_with(
            {'_id': sentinel.taskset_id})
        self.assertItemsEqual(
            ['date_done', 'result', 'task_id'],
            list(ret_val.keys()),
        )

        mock_collection.find_one.return_value = None
        self.backend._restore_group(sentinel.taskset_id)
Ejemplo n.º 6
0
    def test_on_chord_part_return(self, restore):
        tb = CacheBackend(backend='memory://', app=self.app)

        deps = Mock()
        deps.__len__ = Mock()
        deps.__len__.return_value = 2
        restore.return_value = deps
        task = Mock()
        task.name = 'foobarbaz'
        self.app.tasks['foobarbaz'] = task
        task.request.chord = signature(task)

        result = self.app.GroupResult(
            uuid(),
            [self.app.AsyncResult(uuid()) for _ in range(3)],
        )
        task.request.group = result.id
        tb.apply_chord(result, None)

        deps.join_native.assert_not_called()
        tb.on_chord_part_return(task.request, 'SUCCESS', 10)
        deps.join_native.assert_not_called()

        tb.on_chord_part_return(task.request, 'SUCCESS', 10)
        deps.join_native.assert_called_with(propagate=True, timeout=3.0)
        deps.delete.assert_called_with()
Ejemplo n.º 7
0
    def test_poll_result(self):
        with self._result_context() as (results, backend, Message):
            tid = uuid()
            # FFWD's to the latest state.
            state_messages = [
                Message(task_id=tid, status=states.RECEIVED, seq=1),
                Message(task_id=tid, status=states.STARTED, seq=2),
                Message(task_id=tid, status=states.FAILURE, seq=3),
            ]
            for state_message in state_messages:
                results.put(state_message)
            r1 = backend.get_task_meta(tid)
            self.assertDictContainsSubset(
                {'status': states.FAILURE, 'seq': 3}, r1,
                'FFWDs to the last state',
            )

            # Caches last known state.
            tid = uuid()
            results.put(Message(task_id=tid))
            backend.get_task_meta(tid)
            self.assertIn(tid, backend._cache, 'Caches last known state')

            self.assertTrue(state_messages[-1].requeued)

            # Returns cache if no new states.
            results.queue.clear()
            assert not results.qsize()
            backend._cache[tid] = 'hello'
            self.assertEqual(
                backend.get_task_meta(tid), 'hello',
                'Returns cache if no new states',
            )
Ejemplo n.º 8
0
    def test_poll_result(self):
        with self._result_context() as (results, backend, Message):
            tid = uuid()
            # FFWD's to the latest state.
            state_messages = [
                Message(task_id=tid, status=states.RECEIVED, seq=1),
                Message(task_id=tid, status=states.STARTED, seq=2),
                Message(task_id=tid, status=states.FAILURE, seq=3),
            ]
            for state_message in state_messages:
                results.put(state_message)
            r1 = backend.get_task_meta(tid)
            # FFWDs to the last state.
            assert r1['status'] == states.FAILURE
            assert r1['seq'] == 3

            # Caches last known state.
            tid = uuid()
            results.put(Message(task_id=tid))
            backend.get_task_meta(tid)
            assert tid, backend._cache in 'Caches last known state'

            assert state_messages[-1].requeued

            # Returns cache if no new states.
            results.queue.clear()
            assert not results.qsize()
            backend._cache[tid] = 'hello'
            # returns cache if no new states.
            assert backend.get_task_meta(tid) == 'hello'
Ejemplo n.º 9
0
 def test_restore_current_app_fallback(self):
     subs = [MockAsyncResultSuccess(uuid(), app=self.app)]
     ts = self.app.GroupResult(uuid(), subs)
     ts.save()
     with pytest.raises(RuntimeError,
                        message="Test depends on current_app"):
         GroupResult.restore(ts.id)
Ejemplo n.º 10
0
 def setup(self):
     self.simple_message = self.app.amqp.as_task_v2(
         uuid(), 'foo', create_sent_event=True,
     )
     self.simple_message_no_sent_event = self.app.amqp.as_task_v2(
         uuid(), 'foo', create_sent_event=False,
     )
Ejemplo n.º 11
0
 def test_with_parent(self):
     x = self.app.AsyncResult(uuid())
     x.parent = self.app.AsyncResult(uuid())
     y = result_from_tuple(x.as_tuple(), self.app)
     self.assertEqual(y, x)
     self.assertEqual(y.parent, x.parent)
     self.assertIsInstance(y.parent, AsyncResult)
Ejemplo n.º 12
0
 def test_iterate_raises(self):
     ar = MockAsyncResultFailure(uuid(), app=self.app)
     ts = self.app.GroupResult(uuid(), [ar])
     with self.assertPendingDeprecation():
         it = ts.iterate()
     with self.assertRaises(KeyError):
         next(it)
Ejemplo n.º 13
0
    def test_iterate_respects_subpolling_interval(self):
        r1 = self.app.AsyncResult(uuid())
        r2 = self.app.AsyncResult(uuid())
        backend = r1.backend = r2.backend = Mock()
        backend.subpolling_interval = 10

        ready = r1.ready = r2.ready = Mock()

        def se(*args, **kwargs):
            ready.side_effect = KeyError()
            return False
        ready.return_value = False
        ready.side_effect = se

        x = self.app.ResultSet([r1, r2])
        with self.dummy_copy():
            with patch('celery.result.time') as _time:
                with pytest.warns(CPendingDeprecationWarning):
                    with pytest.raises(KeyError):
                        list(x.iterate())
                _time.sleep.assert_called_with(10)

            backend.subpolling_interval = 0
            with patch('celery.result.time') as _time:
                with pytest.warns(CPendingDeprecationWarning):
                    with pytest.raises(KeyError):
                        ready.return_value = False
                        ready.side_effect = se
                        list(x.iterate())
                    _time.sleep.assert_not_called()
Ejemplo n.º 14
0
    def test_add_to_chord(self):
        self.app.backend = Mock(name='backend')

        sig = self.add.s(2, 2)
        sig.delay = Mock(name='sig.delay')
        self.adds.request.group = uuid()
        self.adds.request.id = uuid()

        with pytest.raises(ValueError):
            # task not part of chord
            self.adds.run(sig)
        self.adds.request.chord = self.add.s()

        res1 = self.adds.run(sig, True)
        assert res1 == sig
        assert sig.options['task_id']
        assert sig.options['group_id'] == self.adds.request.group
        assert sig.options['chord'] == self.adds.request.chord
        sig.delay.assert_not_called()
        self.app.backend.add_to_chord.assert_called_with(
            self.adds.request.group, sig.freeze(),
        )

        self.app.backend.reset_mock()
        sig2 = self.add.s(4, 4)
        sig2.delay = Mock(name='sig2.delay')
        res2 = self.adds.run(sig2)
        assert res2 == sig2.delay.return_value
        assert sig2.options['task_id']
        assert sig2.options['group_id'] == self.adds.request.group
        assert sig2.options['chord'] == self.adds.request.chord
        sig2.delay.assert_called_with()
        self.app.backend.add_to_chord.assert_called_with(
            self.adds.request.group, sig2.freeze(),
        )
Ejemplo n.º 15
0
 def test_forget(self):
     subs = [MockAsyncResultSuccess(uuid(), app=self.app),
             MockAsyncResultSuccess(uuid(), app=self.app)]
     ts = self.app.GroupResult(uuid(), subs)
     ts.forget()
     for sub in subs:
         assert sub.forgotten
Ejemplo n.º 16
0
 def test_revoke_from_resultset(self):
     self.app.control.revoke = Mock(name="revoke")
     uuids = [uuid() for _ in range(10)]
     r = self.app.GroupResult(uuid(), [self.app.AsyncResult(x) for x in uuids])
     r.revoke()
     self.app.control.revoke.assert_called_with(
         uuids, connection=None, reply=False, signal=None, terminate=False, timeout=None
     )
Ejemplo n.º 17
0
 def test_join_native_raises(self):
     ts = self.app.GroupResult(uuid(), [self.app.AsyncResult(uuid())])
     ts.iter_native = Mock()
     ts.iter_native.return_value = iter([
         (uuid(), {'status': states.FAILURE, 'result': KeyError()})
     ])
     with pytest.raises(KeyError):
         ts.join_native(propagate=True)
Ejemplo n.º 18
0
 def test_iterate_yields(self):
     ar = MockAsyncResultSuccess(uuid(), app=self.app)
     ar2 = MockAsyncResultSuccess(uuid(), app=self.app)
     ts = self.app.GroupResult(uuid(), [ar, ar2])
     with pytest.warns(CPendingDeprecationWarning):
         it = ts.iterate()
     assert next(it) == 42
     assert next(it) == 42
Ejemplo n.º 19
0
 def test_iterate_eager(self):
     ar1 = EagerResult(uuid(), 42, states.SUCCESS)
     ar2 = EagerResult(uuid(), 42, states.SUCCESS)
     ts = self.app.GroupResult(uuid(), [ar1, ar2])
     with pytest.warns(CPendingDeprecationWarning):
         it = ts.iterate()
     assert next(it) == 42
     assert next(it) == 42
Ejemplo n.º 20
0
 def test_iterate_eager(self):
     ar1 = EagerResult(uuid(), 42, states.SUCCESS)
     ar2 = EagerResult(uuid(), 42, states.SUCCESS)
     ts = self.app.GroupResult(uuid(), [ar1, ar2])
     with self.assertPendingDeprecation():
         it = ts.iterate()
     self.assertEqual(next(it), 42)
     self.assertEqual(next(it), 42)
Ejemplo n.º 21
0
 def test_iterate_yields(self):
     ar = MockAsyncResultSuccess(uuid(), app=self.app)
     ar2 = MockAsyncResultSuccess(uuid(), app=self.app)
     ts = self.app.GroupResult(uuid(), [ar, ar2])
     with self.assertPendingDeprecation():
         it = ts.iterate()
     self.assertEqual(next(it), 42)
     self.assertEqual(next(it), 42)
Ejemplo n.º 22
0
 def test_apply_chord(self):
     tb = CacheBackend(backend='memory://', app=self.app)
     result = self.app.GroupResult(
         uuid(),
         [self.app.AsyncResult(uuid()) for _ in range(3)],
     )
     tb.apply_chord(result, None)
     assert self.app.GroupResult.restore(result.id, backend=tb) == result
Ejemplo n.º 23
0
 def test_GroupResult_with_parent(self):
     parent = self.app.AsyncResult(uuid())
     result = self.app.GroupResult(
         uuid(), [self.app.AsyncResult(uuid()) for _ in range(10)],
         parent
     )
     second_result = result_from_tuple(result.as_tuple(), self.app)
     assert second_result == result
     assert second_result.parent == parent
Ejemplo n.º 24
0
 def test_eq_ne(self):
     ts = self.app.GroupResult(uuid(), [self.app.AsyncResult(uuid())])
     ts2 = self.app.GroupResult(ts.id, ts.results)
     ts3 = self.app.GroupResult(uuid(), [self.app.AsyncResult(uuid())])
     ts4 = self.app.GroupResult(ts.id, [self.app.AsyncResult(uuid())])
     assert ts == ts2
     assert ts != ts3
     assert ts != ts4
     assert ts != object()
Ejemplo n.º 25
0
 def test_eq_ne(self):
     ts = self.app.GroupResult(uuid(), [self.app.AsyncResult(uuid())])
     ts2 = self.app.GroupResult(ts.id, ts.results)
     ts3 = self.app.GroupResult(uuid(), [self.app.AsyncResult(uuid())])
     ts4 = self.app.GroupResult(ts.id, [self.app.AsyncResult(uuid())])
     self.assertEqual(ts, ts2)
     self.assertNotEqual(ts, ts3)
     self.assertNotEqual(ts, ts4)
     self.assertNotEqual(ts, object())
Ejemplo n.º 26
0
    def test_get_many(self):
        data = {uuid(): 'foo', uuid(): 'bar', uuid(): 'baz'}

        tb = FilesystemBackend(app=self.app, url=self.url)
        for key, value in data.items():
            tb.mark_as_done(key, value)

        for key, result in tb.get_many(data.keys()):
            assert result['result'] == data[key]
Ejemplo n.º 27
0
 def test_save_restore_delete_group(self):
     tid = uuid()
     tsr = self.app.GroupResult(
         tid, [self.app.AsyncResult(uuid()) for _ in range(10)],
     )
     self.b.save_group(tid, tsr)
     self.b.restore_group(tid)
     assert self.b.restore_group(tid) == tsr
     self.b.delete_group(tid)
     assert self.b.restore_group(tid) is None
Ejemplo n.º 28
0
 def test_join_native(self):
     backend = SimpleBackend()
     results = [self.app.AsyncResult(uuid(), backend=backend) for i in range(10)]
     ts = self.app.GroupResult(uuid(), results)
     ts.app.backend = backend
     backend.ids = [result.id for result in results]
     res = ts.join_native()
     self.assertEqual(res, list(range(10)))
     callback = Mock(name="callback")
     self.assertFalse(ts.join_native(callback=callback))
     callback.assert_has_calls([call(r.id, i) for i, r in enumerate(ts.results)])
Ejemplo n.º 29
0
 def test_store_result_parent_id(self):
     tid = uuid()
     pid = uuid()
     state = 'SUCCESS'
     result = 10
     request = Context(parent_id=pid)
     self.b.store_result(
         tid, state=state, result=result, request=request,
     )
     stored_meta = self.b.decode(self.b.get(self.b.get_key_for_task(tid)))
     assert stored_meta['parent_id'] == request.parent_id
Ejemplo n.º 30
0
 def test_save_restore_delete_group(self):
     group_id = uuid()
     result_ids = [uuid() for i in range(10)]
     results = list(map(result.AsyncResult, result_ids))
     res = result.GroupResult(group_id, results)
     res.save(backend=self.b)
     saved = result.GroupResult.restore(group_id, backend=self.b)
     assert saved.results == results
     assert saved.id == group_id
     saved.delete(backend=self.b)
     assert result.GroupResult.restore(group_id, backend=self.b) is None
Ejemplo n.º 31
0
 def test_mark_as_done_writes_file(self):
     tb = FilesystemBackend(app=self.app, url=self.url)
     tb.mark_as_done(uuid(), 42)
     assert len(os.listdir(self.directory)) == 1
Ejemplo n.º 32
0
 def test_done_task_is_SUCCESS(self):
     tb = FilesystemBackend(app=self.app, url=self.url)
     tid = uuid()
     tb.mark_as_done(tid, 42)
     assert tb.get_state(tid) == states.SUCCESS
Ejemplo n.º 33
0
 def test_forget_deletes_file(self):
     tb = FilesystemBackend(app=self.app, url=self.url)
     tid = uuid()
     tb.mark_as_done(tid, 42)
     tb.forget(tid)
     assert len(os.listdir(self.directory)) == 0
import pytest

import datetime

from pickle import loads, dumps

from case import ANY, MagicMock, Mock, mock, patch, sentinel, skip
from kombu.exceptions import EncodeError

from celery import uuid
from celery import states
from celery.backends.mongodb import InvalidDocument, MongoBackend
from celery.exceptions import ImproperlyConfigured

COLLECTION = 'taskmeta_celery'
TASK_ID = uuid()
MONGODB_HOST = 'localhost'
MONGODB_PORT = 27017
MONGODB_USER = '******'
MONGODB_PASSWORD = '******'
MONGODB_DATABASE = 'testing'
MONGODB_COLLECTION = 'collection1'
MONGODB_GROUP_COLLECTION = 'group_collection1'


@skip.unless_module('pymongo')
class test_MongoBackend:

    default_url = 'mongodb://*****:*****@hostname.dom/database'
    replica_set_url = (
        'mongodb://*****:*****@hostname.dom,'
Ejemplo n.º 35
0
def longtask():
    task_id = uuid()  # create your task_id
    task = long_task.apply_async(task_id=task_id)
    return jsonify({}), 202, {'Location': url_for('taskstatus', task_id=task.id)}
 def setup(self):
     self.size = 10
     self.ts = self.app.GroupResult(
         uuid(),
         make_mock_group(self.app, self.size),
     )
Ejemplo n.º 37
0
 def test_AsyncResult(self):
     task_id = uuid()
     result = self.retry_task.AsyncResult(task_id)
     assert result.backend == self.retry_task.backend
     assert result.id == task_id
Ejemplo n.º 38
0
 def test_apply_chord(self):
     tb = CacheBackend(backend='memory://', app=self.app)
     gid, res = uuid(), [self.app.AsyncResult(uuid()) for _ in range(3)]
     tb.apply_chord(group(app=self.app), (), gid, {}, result=res)
Ejemplo n.º 39
0
 def test_add_to_chord(self):
     b = self.Backend('redis://', app=self.app)
     gid = uuid()
     b.add_to_chord(gid, 'sig')
     b.client.incr.assert_called_with(b.get_key_for_group(gid, '.t'), 1)
Ejemplo n.º 40
0
 def test_mark_as_started(self):
     tb = DatabaseBackend(self.uri, app=self.app)
     tid = uuid()
     tb.mark_as_started(tid)
     assert tb.get_state(tid) == states.STARTED
Ejemplo n.º 41
0
 def test_mark_as_revoked(self):
     tb = DatabaseBackend(self.uri, app=self.app)
     tid = uuid()
     tb.mark_as_revoked(tid)
     assert tb.get_state(tid) == states.REVOKED
Ejemplo n.º 42
0
def resubmit_jobs(context):
    """
    logic to resubmit the job
    :param context: contents from _context.json
    """

    # iterate through job ids and query to get the job json
    increment_by = None
    new_priority = None
    if "job_priority_increment" in context:
        increment_by = context["job_priority_increment"]
    else:
        new_priority = context["new_job_priority"]

    retry_count_max = context['retry_count_max']

    if isinstance(context['retry_job_id'], list):
        retry_job_ids = context['retry_job_id']
    else:
        retry_job_ids = [context['retry_job_id']]

    for job_id in retry_job_ids:
        print(("Validating retry job: {}".format(job_id)))
        try:
            doc = query_es(job_id)
            if doc['hits']['total']['value'] == 0:
                print('job id %s not found in Elasticsearch. Continuing.' %
                      job_id)
                continue
            doc = doc["hits"]["hits"][0]

            job_json = doc["_source"]["job"]
            task_id = doc["_source"]["uuid"]
            index = doc["_index"]
            _id = doc["_id"]

            # don't retry a retry
            if job_json['type'].startswith('job-lw-mozart-retry'):
                print("Cannot retry retry job %s. Skipping" % job_id)
                continue

            # check retry_remaining_count
            if 'retry_count' in job_json:
                if job_json['retry_count'] < retry_count_max:
                    job_json['retry_count'] = int(job_json['retry_count']) + 1
                else:
                    print(
                        "For job {}, retry_count now is {}, retry_count_max limit of {} reached. Cannot retry again."
                        .format(job_id, job_json['retry_count'],
                                retry_count_max))
                    continue
            else:
                job_json['retry_count'] = 1
            job_json["job_info"]["dedup"] = False

            # clean up job execution info
            for i in ('duration', 'execute_node', 'facts', 'job_dir',
                      'job_url', 'metrics', 'pid', 'public_ip', 'status',
                      'stderr', 'stdout', 'time_end', 'time_queued',
                      'time_start'):
                if i in job_json.get('job_info', {}):
                    del job_json['job_info'][i]

            # set queue time
            job_json['job_info']['time_queued'] = datetime.utcnow().isoformat(
            ) + 'Z'

            # reset priority
            old_priority = job_json['priority']
            job_json['priority'] = get_new_job_priority(
                old_priority=old_priority,
                increment_by=increment_by,
                new_priority=new_priority)

            # get state
            task = app.AsyncResult(task_id)
            state = task.state

            # revoke
            job_id = job_json['job_id']
            try:
                revoke(task_id, state)
                print("revoked original job: %s (%s)" % (job_id, task_id))
            except:
                print("Got error issuing revoke on job %s (%s): %s" %
                      (job_id, task_id, traceback.format_exc()))
                print("Continuing.")

            # generate celery task id
            new_task_id = uuid()
            job_json['task_id'] = new_task_id

            # delete old job status
            delete_by_id(index, _id)

            # log queued status
            job_status_json = {
                'uuid':
                new_task_id,
                'job_id':
                job_id,
                'payload_id':
                job_json['job_info']['job_payload']['payload_task_id'],
                'status':
                'job-queued',
                'job':
                job_json
            }
            log_job_status(job_status_json)

            # submit job
            queue = job_json['job_info']['job_queue']
            run_job.apply_async(
                (job_json, ),
                queue=queue,
                time_limit=job_json['job_info']['time_limit'],
                soft_time_limit=job_json['job_info']['soft_time_limit'],
                priority=job_json['priority'],
                task_id=new_task_id)
        except Exception as ex:
            print("[ERROR] Exception occurred {0}:{1} {2}".format(
                type(ex), ex, traceback.format_exc()),
                  file=sys.stderr)
Ejemplo n.º 43
0
 def setup(self):
     self.app.conf.result_serializer = 'pickle'
     self.tb = CacheBackend(backend='memory://', app=self.app)
     self.tid = uuid()
     self.old_get_best_memcached = backends['memcache']
     backends['memcache'] = lambda: (DummyClient, ensure_bytes)
Ejemplo n.º 44
0
 def create_task_result(self):
     id = uuid()
     taskmeta, created = TaskResult.objects.get_or_create(task_id=id)
     return taskmeta
Ejemplo n.º 45
0
 def test_get_many_times_out(self):
     tasks = [uuid() for _ in range(4)]
     self.b._cache[tasks[1]] = {'status': 'PENDING'}
     with pytest.raises(self.b.TimeoutError):
         list(self.b.get_many(tasks, timeout=0.01, interval=0.01))
Ejemplo n.º 46
0
def start_analyzers(
    analyzers_to_execute,
    analyzers_config,
    runtime_configuration,
    job_id,
    md5,
    is_sample,
):
    set_job_status(job_id, "running")
    if is_sample:
        file_path, filename = get_filepath_filename(job_id)
    else:
        observable_name, observable_classification = get_observable_data(
            job_id)

    for analyzer in analyzers_to_execute:
        ac = analyzers_config[analyzer]
        try:
            module = ac.get("python_module", None)
            if not module:
                raise AnalyzerConfigurationException(
                    f"no python_module available in config for {analyzer} analyzer?!"
                )

            additional_config_params = ac.get("additional_config_params", {})

            adjust_analyzer_config(runtime_configuration,
                                   additional_config_params, analyzer)
            # get celery queue
            queue = ac.get("queue", "default")
            if queue not in CELERY_QUEUES:
                logger.error(
                    f"Analyzer {analyzers_to_execute} has a wrong queue."
                    f" Setting to default")
                queue = "default"
            # construct arguments

            if is_sample:
                # check if we should run the hash instead of the binary
                run_hash = ac.get("run_hash", False)
                if run_hash:
                    # check which kind of hash the analyzer needs
                    run_hash_type = ac.get("run_hash_type", "md5")
                    if run_hash_type == "md5":
                        hash_value = md5
                    elif run_hash_type == "sha256":
                        hash_value = generate_sha256(job_id)
                    else:
                        error_message = (
                            f"only md5 and sha256 are supported "
                            f"but you asked {run_hash_type}. job_id: {job_id}")
                        raise AnalyzerConfigurationException(error_message)
                    # run the analyzer with the hash
                    args = [
                        f"observable_analyzers.{module}",
                        analyzer,
                        job_id,
                        hash_value,
                        "hash",
                        additional_config_params,
                    ]
                else:
                    # run the analyzer with the binary
                    args = [
                        f"file_analyzers.{module}",
                        analyzer,
                        job_id,
                        file_path,
                        filename,
                        md5,
                        additional_config_params,
                    ]
            else:
                # observables analyzer case
                args = [
                    f"observable_analyzers.{module}",
                    analyzer,
                    job_id,
                    observable_name,
                    observable_classification,
                    additional_config_params,
                ]
            # run analyzer with a celery task asynchronously
            stl = ac.get("soft_time_limit", 300)
            t_id = uuid()
            celery_app.send_task(
                "run_analyzer",
                args=args,
                queue=queue,
                soft_time_limit=stl,
                task_id=t_id,
            )
            # to track task_id by job_id
            task_ids = cache.get(job_id)
            if isinstance(task_ids, list):
                task_ids.append(t_id)
            else:
                task_ids = [t_id]
            cache.set(job_id, task_ids)

        except (AnalyzerConfigurationException, AnalyzerRunException) as e:
            err_msg = f"({analyzer}, job_id #{job_id}) -> Error: {e}"
            logger.error(err_msg)
            set_failed_analyzer(analyzer, job_id, err_msg)
Ejemplo n.º 47
0
 def create_group_result(self):
     id = uuid()
     taskmeta, created = GroupResult.objects.get_or_create(group_id=id)
     return taskmeta