def failure(job=None, exc=None):
# depending on the failure we might requeue it with a
# delay, using job.enqueue()
# XXX
pl = job.redis.pipeline()
job.metadata['ended'] = time.time()
save_job(job)
dump = json.dumps({'msg': 'Error', 'data': str(exc)})
console_key = 'retools:jobconsole:%s' % job.job_id
current = job.redis.get(console_key)
if current is not None:
data = current + str(exc)
else:
data = str(exc)
pl.set(console_key, data)
pl.srem('retools:started', job.job_id)
pl.delete('retools:jobpid:%s' % str(os.getpid()))
pl.sadd('retools:queue:failures', job.job_id)
pl.lpush('retools:result:%s' % job.job_id, dump)
pl.sadd('retools:consoles', job.job_id)
pl.expire('retools:result:%s', 3600)
nodes = os.environ.get('MARTEAU_NODES')
if nodes is not None:
nodes = nodes.split(',')
for name in nodes:
data = job.redis.get('retools:node:%s' % name)
node = Node(**json.loads(data))
node.status = 'idle'
names = job.redis.smembers('retools:nodes')
if node.name not in names:
job.redis.sadd('retools:nodes', node.name)
job.redis.set('retools:node:%s' % node.name, node.to_json())
pl.execute()
def success(job=None, result=None):
pl = job.redis.pipeline()
job.metadata['ended'] = time.time()
save_job(job)
result = json.dumps({'data': result, 'msg': 'Success'})
pl.srem('retools:started', job.job_id)
pl.delete('retools:jobpid:%s' % str(os.getpid()))
pl.sadd('retools:queue:successes', job.job_id)
pl.lpush('retools:result:%s' % job.job_id, result)
pl.expire('retools:result:%s', 3600)
pl.sadd('retools:consoles', job.job_id)
nodes = os.environ.get('MARTEAU_NODES')
if nodes is not None:
nodes = nodes.split(',')
for name in nodes:
data = job.redis.get('retools:node:%s' % name)
if data is None:
continue
node = Node(**json.loads(data))
node.status = 'idle'
names = job.redis.smembers('retools:nodes')
if node.name not in names:
job.redis.sadd('retools:nodes', node.name)
job.redis.set('retools:node:%s' % node.name, node.to_json())
pl.execute()
def success(job=None, result=None):
pl = job.redis.pipeline()
job.metadata['ended'] = time.time()
save_job(job)
result = json.dumps({'data': result, 'msg': 'Success'})
pl.srem('retools:started', job.job_id)
pl.delete('retools:jobpid:%s' % str(os.getpid()))
pl.sadd('retools:queue:successes', job.job_id)
pl.lpush('retools:result:%s' % job.job_id, result)
pl.expire('retools:result:%s', 3600)
pl.sadd('retools:consoles', job.job_id)
nodes = os.environ.get('MARTEAU_NODES')
if nodes is not None:
nodes = nodes.split(',')
for name in nodes:
data = job.redis.get('retools:node:%s' % name)
if data is None:
continue
node = Node(**json.loads(data))
node.status = 'idle'
names = job.redis.smembers('retools:nodes')
if node.name not in names:
job.redis.sadd('retools:nodes', node.name)
job.redis.set('retools:node:%s' % node.name, node.to_json())
pl.execute()
def get_nodes(self, check_available=False):
names = self._conn.smembers('retools:nodes')
nodes = []
for name in sorted(names):
node = self._conn.get('retools:node:%s' % name)
node = Node(**json.loads(node))
if check_available and (node.status != 'idle' or not node.enabled):
continue
nodes.append(node)
return nodes
def failure(job=None, exc=None):
# depending on the failure we might requeue it with a
# delay, using job.enqueue()
pl = job.redis.pipeline()
job.metadata['ended'] = time.time()
save_job(job)
dump = json.dumps({'msg': 'Error', 'data': str(exc)})
console_key = 'retools:jobconsole:%s' % job.job_id
current = job.redis.get(console_key)
if current is not None:
data = current + str(exc) + '\n'
else:
data = str(exc) + '\n'
pl.set(console_key, data)
pl.srem('retools:started', job.job_id)
pl.delete('retools:jobpid:%s' % str(os.getpid()))
pl.sadd('retools:queue:failures', job.job_id)
pl.lpush('retools:result:%s' % job.job_id, dump)
pl.sadd('retools:consoles', job.job_id)
pl.expire('retools:result:%s', 3600)
nodes = os.environ.get('MARTEAU_NODES')
if nodes is not None:
nodes = nodes.split(',')
for name in nodes:
data = job.redis.get('retools:node:%s' % name)
if data is None:
continue
node = Node(**json.loads(data))
node.status = 'idle'
names = job.redis.smembers('retools:nodes')
if node.name not in names:
job.redis.sadd('retools:nodes', node.name)
job.redis.set('retools:node:%s' % node.name, node.to_json())
if 'enough free nodes' in str(exc):
job.enqueue()
pl.execute()
def add_node(request):
"""Adds a new into Marteau"""
check_auth(request)
owner = request.user
form = Form(request, schema=NodeSchema)
if not form.validate():
request.session.flash("Bad node name")
return HTTPFound(location='/nodes')
node_name = form.data.get('name')
node = Node(name=node_name, owner=owner)
queue = request.registry['queue']
queue.save_node(node)
return HTTPFound(location='/nodes')
def create_nodes(self,
image_id,
count,
instance_type='t1.micro',
security_groups=['marteau'],
key_name='tarek'):
min_count = max_count = count
reservation = self.conn.run_instances(image_id=image_id,
instance_type=instance_type,
security_groups=security_groups,
key_name=key_name,
min_count=count,
max_count=count)
for instance in reservation.instances:
while instance.state != 'running':
time.sleep(5)
instance.update()
return [
Node(name=instance.public_dns_name, id=instance.id)
for instance in reservation.instances
]
def test_attr(self):
node = Node(name='ok', a=1, b=2)
self.assertEqual(node.a, 1)
def get_node(self, name):
data = self._conn.get('retools:node:%s' % name)
return Node(**json.loads(data))