def test_regular_task(self):
T1 = self.createTaskCls("T1", "c.unittest.t.t1")
self.assertTrue(isinstance(T1(), T1))
self.assertTrue(T1().run())
self.assertTrue(callable(T1()),
"Task class is callable()")
self.assertTrue(T1()(),
"Task class runs run() when called")
# task name generated out of class module + name.
T2 = self.createTaskCls("T2")
self.assertEquals(T2().name, "celery.tests.test_task.T2")
registry.tasks.register(T1)
t1 = T1()
consumer = t1.get_consumer()
self.assertRaises(NotImplementedError, consumer.receive, "foo", "foo")
consumer.discard_all()
self.assertTrue(consumer.fetch() is None)
# Without arguments.
presult = t1.delay()
self.assertNextTaskDataEquals(consumer, presult, t1.name)
# With arguments.
presult2 = task.delay_task(t1.name, name="George Constanza")
self.assertNextTaskDataEquals(consumer, presult2, t1.name,
name="George Constanza")
# With eta.
presult2 = task.apply_async(t1, kwargs=dict(name="George Constanza"),
eta=datetime.now() + timedelta(days=1))
self.assertNextTaskDataEquals(consumer, presult2, t1.name,
name="George Constanza", test_eta=True)
# With countdown.
presult2 = task.apply_async(t1, kwargs=dict(name="George Constanza"),
countdown=10)
self.assertNextTaskDataEquals(consumer, presult2, t1.name,
name="George Constanza", test_eta=True)
self.assertRaises(registry.tasks.NotRegistered, task.delay_task,
"some.task.that.should.never.exist.X.X.X.X.X")
# Discarding all tasks.
task.discard_all()
tid3 = task.delay_task(t1.name)
self.assertEquals(task.discard_all(), 1)
self.assertTrue(consumer.fetch() is None)
self.assertFalse(task.is_done(presult.task_id))
self.assertFalse(presult.is_done())
default_backend.mark_as_done(presult.task_id, result=None)
self.assertTrue(task.is_done(presult.task_id))
self.assertTrue(presult.is_done())
publisher = t1.get_publisher()
self.assertTrue(isinstance(publisher, messaging.TaskPublisher))
def test_regular_task(self):
T1 = self.createTaskCls("T1", "c.unittest.t.t1")
self.assertIsInstance(T1(), T1)
self.assertTrue(T1().run())
self.assertTrue(callable(T1()),
"Task class is callable()")
self.assertTrue(T1()(),
"Task class runs run() when called")
# task name generated out of class module + name.
T2 = self.createTaskCls("T2")
self.assertTrue(T2().name.endswith("test_task.T2"))
t1 = T1()
consumer = t1.get_consumer()
self.assertRaises(NotImplementedError, consumer.receive, "foo", "foo")
consumer.discard_all()
self.assertIsNone(consumer.fetch())
# Without arguments.
presult = t1.delay()
self.assertNextTaskDataEqual(consumer, presult, t1.name)
# With arguments.
presult2 = t1.apply_async(kwargs=dict(name="George Constanza"))
self.assertNextTaskDataEqual(consumer, presult2, t1.name,
name="George Constanza")
# send_task
sresult = send_task(t1.name, kwargs=dict(name="Elaine M. Benes"))
self.assertNextTaskDataEqual(consumer, sresult, t1.name,
name="Elaine M. Benes")
# With eta.
presult2 = task.apply_async(t1, kwargs=dict(name="George Constanza"),
eta=datetime.now() + timedelta(days=1))
self.assertNextTaskDataEqual(consumer, presult2, t1.name,
name="George Constanza", test_eta=True)
# With countdown.
presult2 = task.apply_async(t1, kwargs=dict(name="George Constanza"),
countdown=10)
self.assertNextTaskDataEqual(consumer, presult2, t1.name,
name="George Constanza", test_eta=True)
# Discarding all tasks.
consumer.discard_all()
task.apply_async(t1)
self.assertEqual(consumer.discard_all(), 1)
self.assertIsNone(consumer.fetch())
self.assertFalse(presult.successful())
default_backend.mark_as_done(presult.task_id, result=None)
self.assertTrue(presult.successful())
publisher = t1.get_publisher()
self.assertIsInstance(publisher, messaging.TaskPublisher)
def _applier(request, **options):
kwargs = kwdict(request.method == 'POST' and request.POST
or request.GET)
# no multivalue
kwargs = dict(((k, v) for k, v in kwargs.iteritems()), **options)
result = task.apply_async(kwargs=kwargs)
return JsonResponse({'ok': 'true', 'task_id': result.task_id})
def _applier(request, **options):
kwargs = kwdict(request.method == "POST" and \
request.POST.copy() or request.GET.copy())
kwargs.update(options)
result = task.apply_async(kwargs=kwargs)
return JsonResponse({"ok": "true", "task_id": result.task_id})
def _applier(request, **options):
kwargs = kwdict(request.method == 'POST' and
request.POST or request.GET)
# no multivalue
kwargs = dict(((k, v) for k, v in kwargs.iteritems()), **options)
result = task.apply_async(kwargs=kwargs)
return JsonResponse({'ok': 'true', 'task_id': result.task_id})
def _applier(request, **options):
kwargs = kwdict(request.method == "POST" and \
request.POST or request.GET)
# no multivalue
kwargs = dict(((k, v) for k, v in kwargs.iteritems()), **options)
result = task.apply_async(kwargs=kwargs)
return JsonResponse({"ok": "true", "task_id": result.task_id})
def _applier(request, **options):
kwargs = request.POST if request.method == 'POST' else request.GET
# no multivalue
kwargs = {k: v for k, v in items(kwargs)}
if options:
kwargs.update(options)
result = task.apply_async(kwargs=kwargs)
return JsonResponse({'ok': 'true', 'task_id': result.task_id})
def _applier(request, **options):
kwargs = kwdict(request.method == "POST" and \
request.POST or request.GET)
kwargs = dict((k, v[0])
for k, v in dict(kwargs, **options).iteritems())
result = task.apply_async(kwargs=kwargs)
return JsonResponse({"ok": "true", "task_id": result.task_id})
def _applier(request, **options):
kwargs = request.POST if request.method == "POST" else request.GET
# no multivalue
kwargs = dict(((k, v) for k, v in kwargs.items()), **options)
if options:
kwargs.update(options)
result = task.apply_async(kwargs=kwargs)
return JsonResponse({"ok": "true", "task_id": result.task_id})
def _applier(request, **options):
kwargs = request.method == "POST" and \
request.POST.copy() or request.GET.copy()
kwargs = dict((key.encode("utf-8"), value)
for key, value in kwargs.items())
kwargs.update(options)
result = task.apply_async(kwargs=kwargs)
response_data = {"ok": "true", "task_id": result.task_id}
return HttpResponse(JSON_dump(response_data),
mimetype="application/json")
def apply(request, task_name, *args):
"""View applying a task.
Example:
http://e.com/celery/apply/task_name/arg1/arg2//?kwarg1=a&kwarg2=b
**NOTE** Use with caution, preferably not make this publicly accessible
without ensuring your code is safe!
"""
kwargs = request.method == "POST" and \
request.POST.copy() or request.GET.copy()
kwargs = dict((key.encode("utf-8"), value)
for key, value in kwargs.items())
if task_name not in tasks:
raise Http404("apply: no such task")
task = tasks[task_name]
result = apply_async(task, args=args, kwargs=kwargs)
return JSON_dump({"ok": "true", "task_id": result.task_id})
def start_l3a_job(imagery_list, job_id, aoi_name, window_size=3, maja_ver="3.2.2"):
"""Given the list of input imagery, start the job processes
Major Steps:
1. Generate date array from input imagery list
2. Generate command for each date from 0 to n
3. Run command for date 0
3. Check if expected L2A output exists on S3 bucket, if not:
3. Retrieve l1c imagery for date 0 from S3 bucket, if l1c imagery cannot be found, entire job fails, extract zip
3. Run demmaccs.py for date 0
3. Upload l2a output to S3 bucket
3. Delete local working dir contents
4. Run next command for date 1 to n
4. Check if expected L2A exists on S3 bucket, if not:
4. Retrieve l1c imagery for date i from S3 bucket, if l1c imagery cannot be found, entire job fails, extract zip
4. Retrieve l2a imagery for date i-1, date i-2, date i-3
3. Run demmaccs.py for date i
3. Upload l2a output to S3 bucket
4. Delete local working dir contents
5. If all commands complete successfully, report job complete
6. Else report job failure
imagery list format example:
JSON:
"parameters": {
"aoi_name": "Test Area 2",
"window_size": 3,
"imagery_list": {
"sentinel2": {
"20190613": [
"S2A_MSIL1C_20190613T182921_N0207_R027_T12UUA_20190613T220508"
],
"20190616": [],
"20190618": [
"S2B_MSIL1C_20190618T182929_N0207_R027_T12UUA_20190618T220500"
],
"20190621": [],
"20190623": [],
"20190626": [],
"20190628": [
"S2B_MSIL1C_20190628T182929_N0207_R027_T12UUA_20190628T221748"
],
"20190701": [],
"20190703": [
"S2A_MSIL1C_20190703T182921_N0207_R027_T12UUA_20190703T220857"
],
"20190706": [],
"20190718": [
"S2B_MSIL1C_20190718T182929_N0208_R027_T12UUA_20190718T220349"
],
"20190721": []
}
}
"""
conv_imagery_list = {}
imagery_list = imagery_list["sentinel2"]
for (key, value) in imagery_list.items():
if len(value) > 0:
conv_imagery_list[key] = []
for img in value:
# S2A_MSIL1C_20190703T182921_N0207_R027_T12UUA_20190703T2328
conv_imagery_list[key].append(img)
print(conv_imagery_list)
# Todo: add checks for existing products (to prevent redundant work)
task = generate_l3a.s(conv_imagery_list, window_size, aoi_name)
logging.info(task)
task_result = task.apply_async().get()
json_result = json.dumps(task_result)
return json_result
def _applier(request, **options):
kwargs = kwdict(request.method == "POST" and request.POST.copy() or request.GET.copy())
kwargs.update(options)
result = task.apply_async(kwargs=kwargs)
return JsonResponse({"ok": "true", "task_id": result.task_id})
def test_regular_task(self):
T1 = self.createTaskCls("T1", "c.unittest.t.t1")
self.assertIsInstance(T1(), T1)
self.assertTrue(T1().run())
self.assertTrue(callable(T1()), "Task class is callable()")
self.assertTrue(T1()(), "Task class runs run() when called")
# task name generated out of class module + name.
T2 = self.createTaskCls("T2")
self.assertTrue(T2().name.endswith("test_task.T2"))
t1 = T1()
consumer = t1.get_consumer()
self.assertRaises(NotImplementedError, consumer.receive, "foo", "foo")
consumer.discard_all()
self.assertIsNone(consumer.fetch())
# Without arguments.
presult = t1.delay()
self.assertNextTaskDataEqual(consumer, presult, t1.name)
# With arguments.
presult2 = t1.apply_async(kwargs=dict(name="George Costanza"))
self.assertNextTaskDataEqual(consumer,
presult2,
t1.name,
name="George Costanza")
# send_task
sresult = send_task(t1.name, kwargs=dict(name="Elaine M. Benes"))
self.assertNextTaskDataEqual(consumer,
sresult,
t1.name,
name="Elaine M. Benes")
# With eta.
presult2 = task.apply_async(t1,
kwargs=dict(name="George Costanza"),
eta=datetime.now() + timedelta(days=1))
self.assertNextTaskDataEqual(consumer,
presult2,
t1.name,
name="George Costanza",
test_eta=True)
# With countdown.
presult2 = task.apply_async(t1,
kwargs=dict(name="George Costanza"),
countdown=10)
self.assertNextTaskDataEqual(consumer,
presult2,
t1.name,
name="George Costanza",
test_eta=True)
# Discarding all tasks.
consumer.discard_all()
task.apply_async(t1)
self.assertEqual(consumer.discard_all(), 1)
self.assertIsNone(consumer.fetch())
self.assertFalse(presult.successful())
default_backend.mark_as_done(presult.task_id, result=None)
self.assertTrue(presult.successful())
publisher = t1.get_publisher()
self.assertIsInstance(publisher, messaging.TaskPublisher)
