def test_regular_task(self): T1 = self.createTaskCls("T1", "c.unittest.t.t1") self.assertTrue(isinstance(T1(), T1)) self.assertTrue(T1().run()) self.assertTrue(callable(T1()), "Task class is callable()") self.assertTrue(T1()(), "Task class runs run() when called") # task name generated out of class module + name. T2 = self.createTaskCls("T2") self.assertEquals(T2().name, "celery.tests.test_task.T2") registry.tasks.register(T1) t1 = T1() consumer = t1.get_consumer() self.assertRaises(NotImplementedError, consumer.receive, "foo", "foo") consumer.discard_all() self.assertTrue(consumer.fetch() is None) # Without arguments. presult = t1.delay() self.assertNextTaskDataEquals(consumer, presult, t1.name) # With arguments. presult2 = task.delay_task(t1.name, name="George Constanza") self.assertNextTaskDataEquals(consumer, presult2, t1.name, name="George Constanza") # With eta. presult2 = task.apply_async(t1, kwargs=dict(name="George Constanza"), eta=datetime.now() + timedelta(days=1)) self.assertNextTaskDataEquals(consumer, presult2, t1.name, name="George Constanza", test_eta=True) # With countdown. presult2 = task.apply_async(t1, kwargs=dict(name="George Constanza"), countdown=10) self.assertNextTaskDataEquals(consumer, presult2, t1.name, name="George Constanza", test_eta=True) self.assertRaises(registry.tasks.NotRegistered, task.delay_task, "some.task.that.should.never.exist.X.X.X.X.X") # Discarding all tasks. task.discard_all() tid3 = task.delay_task(t1.name) self.assertEquals(task.discard_all(), 1) self.assertTrue(consumer.fetch() is None) self.assertFalse(task.is_done(presult.task_id)) self.assertFalse(presult.is_done()) default_backend.mark_as_done(presult.task_id, result=None) self.assertTrue(task.is_done(presult.task_id)) self.assertTrue(presult.is_done()) publisher = t1.get_publisher() self.assertTrue(isinstance(publisher, messaging.TaskPublisher))
def test_regular_task(self): T1 = self.createTaskCls("T1", "c.unittest.t.t1") self.assertIsInstance(T1(), T1) self.assertTrue(T1().run()) self.assertTrue(callable(T1()), "Task class is callable()") self.assertTrue(T1()(), "Task class runs run() when called") # task name generated out of class module + name. T2 = self.createTaskCls("T2") self.assertTrue(T2().name.endswith("test_task.T2")) t1 = T1() consumer = t1.get_consumer() self.assertRaises(NotImplementedError, consumer.receive, "foo", "foo") consumer.discard_all() self.assertIsNone(consumer.fetch()) # Without arguments. presult = t1.delay() self.assertNextTaskDataEqual(consumer, presult, t1.name) # With arguments. presult2 = t1.apply_async(kwargs=dict(name="George Constanza")) self.assertNextTaskDataEqual(consumer, presult2, t1.name, name="George Constanza") # send_task sresult = send_task(t1.name, kwargs=dict(name="Elaine M. Benes")) self.assertNextTaskDataEqual(consumer, sresult, t1.name, name="Elaine M. Benes") # With eta. presult2 = task.apply_async(t1, kwargs=dict(name="George Constanza"), eta=datetime.now() + timedelta(days=1)) self.assertNextTaskDataEqual(consumer, presult2, t1.name, name="George Constanza", test_eta=True) # With countdown. presult2 = task.apply_async(t1, kwargs=dict(name="George Constanza"), countdown=10) self.assertNextTaskDataEqual(consumer, presult2, t1.name, name="George Constanza", test_eta=True) # Discarding all tasks. consumer.discard_all() task.apply_async(t1) self.assertEqual(consumer.discard_all(), 1) self.assertIsNone(consumer.fetch()) self.assertFalse(presult.successful()) default_backend.mark_as_done(presult.task_id, result=None) self.assertTrue(presult.successful()) publisher = t1.get_publisher() self.assertIsInstance(publisher, messaging.TaskPublisher)
def _applier(request, **options): kwargs = kwdict(request.method == 'POST' and request.POST or request.GET) # no multivalue kwargs = dict(((k, v) for k, v in kwargs.iteritems()), **options) result = task.apply_async(kwargs=kwargs) return JsonResponse({'ok': 'true', 'task_id': result.task_id})
def _applier(request, **options): kwargs = kwdict(request.method == "POST" and \ request.POST.copy() or request.GET.copy()) kwargs.update(options) result = task.apply_async(kwargs=kwargs) return JsonResponse({"ok": "true", "task_id": result.task_id})
def _applier(request, **options): kwargs = kwdict(request.method == "POST" and \ request.POST or request.GET) # no multivalue kwargs = dict(((k, v) for k, v in kwargs.iteritems()), **options) result = task.apply_async(kwargs=kwargs) return JsonResponse({"ok": "true", "task_id": result.task_id})
def _applier(request, **options): kwargs = request.POST if request.method == 'POST' else request.GET # no multivalue kwargs = {k: v for k, v in items(kwargs)} if options: kwargs.update(options) result = task.apply_async(kwargs=kwargs) return JsonResponse({'ok': 'true', 'task_id': result.task_id})
def _applier(request, **options): kwargs = kwdict(request.method == "POST" and \ request.POST or request.GET) kwargs = dict((k, v[0]) for k, v in dict(kwargs, **options).iteritems()) result = task.apply_async(kwargs=kwargs) return JsonResponse({"ok": "true", "task_id": result.task_id})
def _applier(request, **options): kwargs = request.POST if request.method == "POST" else request.GET # no multivalue kwargs = dict(((k, v) for k, v in kwargs.items()), **options) if options: kwargs.update(options) result = task.apply_async(kwargs=kwargs) return JsonResponse({"ok": "true", "task_id": result.task_id})
def _applier(request, **options): kwargs = request.method == "POST" and \ request.POST.copy() or request.GET.copy() kwargs = dict((key.encode("utf-8"), value) for key, value in kwargs.items()) kwargs.update(options) result = task.apply_async(kwargs=kwargs) response_data = {"ok": "true", "task_id": result.task_id} return HttpResponse(JSON_dump(response_data), mimetype="application/json")
def apply(request, task_name, *args): """View applying a task. Example: http://e.com/celery/apply/task_name/arg1/arg2//?kwarg1=a&kwarg2=b **NOTE** Use with caution, preferably not make this publicly accessible without ensuring your code is safe! """ kwargs = request.method == "POST" and \ request.POST.copy() or request.GET.copy() kwargs = dict((key.encode("utf-8"), value) for key, value in kwargs.items()) if task_name not in tasks: raise Http404("apply: no such task") task = tasks[task_name] result = apply_async(task, args=args, kwargs=kwargs) return JSON_dump({"ok": "true", "task_id": result.task_id})
def start_l3a_job(imagery_list, job_id, aoi_name, window_size=3, maja_ver="3.2.2"): """Given the list of input imagery, start the job processes Major Steps: 1. Generate date array from input imagery list 2. Generate command for each date from 0 to n 3. Run command for date 0 3. Check if expected L2A output exists on S3 bucket, if not: 3. Retrieve l1c imagery for date 0 from S3 bucket, if l1c imagery cannot be found, entire job fails, extract zip 3. Run demmaccs.py for date 0 3. Upload l2a output to S3 bucket 3. Delete local working dir contents 4. Run next command for date 1 to n 4. Check if expected L2A exists on S3 bucket, if not: 4. Retrieve l1c imagery for date i from S3 bucket, if l1c imagery cannot be found, entire job fails, extract zip 4. Retrieve l2a imagery for date i-1, date i-2, date i-3 3. Run demmaccs.py for date i 3. Upload l2a output to S3 bucket 4. Delete local working dir contents 5. If all commands complete successfully, report job complete 6. Else report job failure imagery list format example: JSON: "parameters": { "aoi_name": "Test Area 2", "window_size": 3, "imagery_list": { "sentinel2": { "20190613": [ "S2A_MSIL1C_20190613T182921_N0207_R027_T12UUA_20190613T220508" ], "20190616": [], "20190618": [ "S2B_MSIL1C_20190618T182929_N0207_R027_T12UUA_20190618T220500" ], "20190621": [], "20190623": [], "20190626": [], "20190628": [ "S2B_MSIL1C_20190628T182929_N0207_R027_T12UUA_20190628T221748" ], "20190701": [], "20190703": [ "S2A_MSIL1C_20190703T182921_N0207_R027_T12UUA_20190703T220857" ], "20190706": [], "20190718": [ "S2B_MSIL1C_20190718T182929_N0208_R027_T12UUA_20190718T220349" ], "20190721": [] } } """ conv_imagery_list = {} imagery_list = imagery_list["sentinel2"] for (key, value) in imagery_list.items(): if len(value) > 0: conv_imagery_list[key] = [] for img in value: # S2A_MSIL1C_20190703T182921_N0207_R027_T12UUA_20190703T2328 conv_imagery_list[key].append(img) print(conv_imagery_list) # Todo: add checks for existing products (to prevent redundant work) task = generate_l3a.s(conv_imagery_list, window_size, aoi_name) logging.info(task) task_result = task.apply_async().get() json_result = json.dumps(task_result) return json_result
def _applier(request, **options): kwargs = kwdict(request.method == "POST" and request.POST.copy() or request.GET.copy()) kwargs.update(options) result = task.apply_async(kwargs=kwargs) return JsonResponse({"ok": "true", "task_id": result.task_id})
def test_regular_task(self): T1 = self.createTaskCls("T1", "c.unittest.t.t1") self.assertIsInstance(T1(), T1) self.assertTrue(T1().run()) self.assertTrue(callable(T1()), "Task class is callable()") self.assertTrue(T1()(), "Task class runs run() when called") # task name generated out of class module + name. T2 = self.createTaskCls("T2") self.assertTrue(T2().name.endswith("test_task.T2")) t1 = T1() consumer = t1.get_consumer() self.assertRaises(NotImplementedError, consumer.receive, "foo", "foo") consumer.discard_all() self.assertIsNone(consumer.fetch()) # Without arguments. presult = t1.delay() self.assertNextTaskDataEqual(consumer, presult, t1.name) # With arguments. presult2 = t1.apply_async(kwargs=dict(name="George Costanza")) self.assertNextTaskDataEqual(consumer, presult2, t1.name, name="George Costanza") # send_task sresult = send_task(t1.name, kwargs=dict(name="Elaine M. Benes")) self.assertNextTaskDataEqual(consumer, sresult, t1.name, name="Elaine M. Benes") # With eta. presult2 = task.apply_async(t1, kwargs=dict(name="George Costanza"), eta=datetime.now() + timedelta(days=1)) self.assertNextTaskDataEqual(consumer, presult2, t1.name, name="George Costanza", test_eta=True) # With countdown. presult2 = task.apply_async(t1, kwargs=dict(name="George Costanza"), countdown=10) self.assertNextTaskDataEqual(consumer, presult2, t1.name, name="George Costanza", test_eta=True) # Discarding all tasks. consumer.discard_all() task.apply_async(t1) self.assertEqual(consumer.discard_all(), 1) self.assertIsNone(consumer.fetch()) self.assertFalse(presult.successful()) default_backend.mark_as_done(presult.task_id, result=None) self.assertTrue(presult.successful()) publisher = t1.get_publisher() self.assertIsInstance(publisher, messaging.TaskPublisher)