def test_create_and_get_task(self):
task = Task.create(name='foo', inputs=['foo','bar'])
self.assertEquals('foo', task.name)
self.assertEquals(2, len(task.inputs))
self.assertTrue(task.is_processing())
gotten = Task.get(task.id)
self.assertEquals('foo', gotten.name)
self.assertEquals(2, len(gotten.inputs))
self.assertTrue(task.is_processing())
def create_task():
form = TaskForm(request.form)
if not form.validate():
return error(form.errors)
response = requests.get(form.url.data)
task = Task.create(name=form.name.data, inputs=response.json())
return created(task)
def start_mosaic(url='http://25.media.tumblr.com/tumblr_mb50xf8S9c1rgw39go1_500.jpg'):
response = requests.get(url)
image = Image.open(StringIO(response.content))
width, height = image.size
inputs = []
for x in range(0, width, RESOLUTION):
for y in xrange(0, height, RESOLUTION):
tile = image.crop(box=(x, y, x + RESOLUTION, y + RESOLUTION))
r = g = b = 0.0
for pixel in tile.getdata():
try:
r += pixel[0]
g += pixel[1]
b += pixel[2]
except IndexError:
r += pixel
g += pixel
b += pixel
num_pixels = float(len(tile.getdata()))
r,g,b = r/num_pixels, g/num_pixels, b/num_pixels
inputs.append(json.dumps({'x':x, 'y':y, 'r':r, 'g':g, 'b':b}))
models.create_redis(app, db=2)
shuffle(inputs)
task = Task.create(name='mosaic', inputs=inputs)
processes = multiprocessing.cpu_count() or 1
pool = multiprocessing.Pool(processes)
pool.map(_work_mosaic, [task] * processes)
def echo_socket(ws):
ws.receive()
task = Task.get_last()
while task.is_processing():
result = task.get_task_result()
ws.send(dumps({'url':result}))
ws.close()
def test_worker_no_requests(self):
task = Task.create(name='foo', inputs=[])
worker = Worker(task)
worker.work()
self.assertEquals(0, len(task.get_requests()))
self.assertEquals(0, len(task.get_results()))
def aww():
t = Task.create(name='test', inputs=[
'http://i.imgur.com/zOEilgl.jpg',
'http://i.imgur.com/wDRinK6.jpg',
'http://i.imgur.com/f5uhRgN.jpg',
'http://i.imgur.com/xSgkwpO.jpg?1'])
processes = multiprocessing.cpu_count() or 1
pool = multiprocessing.Pool(processes)
pool.map(_work, [t] * processes)
def test_worker_some_requests(self):
task = Task.create(name='foo', inputs=['foo', 'bar'])
worker = Worker(task)
worker.work()
self.assertEquals(0, len(task.get_requests()))
results = task.get_results()
self.assertEquals(2, len(results))
self.assertIn('foo', results)
self.assertIn('bar', results)
def test_create_task(self):
data = {
'name' : 'foo',
'url' : 'https://s3.amazonaws.com/sonofjscompute-testing/sample.json'
}
resp = self.client.post('/tasks/', data=data)
self.assertEquals(201, resp.status_code)
json = loads(resp.data)
self.assertTrue(json['id'])
self.assertEquals('foo', json['name'])
inputs = Task.get(json['id']).get_requests()
self.assertEquals(2, len(inputs))
self.assertEquals('foo', inputs[0])
self.assertEquals('bar', inputs[1])
def test_add_and_get_task_result(self):
task = Task.create(name='foo', inputs=['foo', 'bar'])
self.assertEquals(2, len(task.get_requests()))
task.add_task_result('baz')
self.assertEquals(2, len(task.get_requests()))
self.assertEquals(1, len(task.get_results()))
self.assertEquals('-1',
self.redis.hget('tasks:%d' % task.id, 'processing'))
result = task.get_task_result()
self.assertEquals('baz', result)
self.assertEquals(2, len(task.get_requests()))
self.assertEquals(0, len(task.get_results()))
self.assertEquals('-1',
self.redis.hget('tasks:%d' % task.id, 'processing'))
def test_get_task_request(self):
task = Task.create(name='foo', inputs=['foo', 'bar'])
self.assertEquals(2, len(task.get_requests()))
self.assertTrue(task.is_processing())
request = task.get_task_request()
self.assertEquals('foo', request)
self.assertTrue(task.is_processing())
self.assertEquals(1,
models.redis.llen('tasks:%d:requests-queue' % task.id))
self.assertEquals('1',
self.redis.hget('tasks:%d' % task.id, 'processing'))
self.assertTrue(task.is_processing())
self.assertEquals(1, len(task.get_requests()))
request = task.get_task_request()
self.assertEquals('bar', request)
self.assertTrue(task.is_processing())
self.assertEquals(0, len(task.get_requests()))
def test_remaining_count(self):
task = Task.create(name='foo', inputs=[])
self.assertEquals(0, task.remaining_count())
task = Task.create(name='foo', inputs=['foo','bar'])
self.assertEquals(2, task.remaining_count())
def run_worker(task_id):
task_id = int(task_id)
task = Task.get(task_id)
processes = multiprocessing.cpu_count() or 1
pool = multiprocessing.Pool(processes)
pool.map(_work, [task] * processes)
