示例#1
0
文件: queue.py 项目: whitmo/marteau
def failure(job=None, exc=None):
    # depending on the failure we might requeue it with a
    # delay, using job.enqueue()
    # XXX
    pl = job.redis.pipeline()
    job.metadata['ended'] = time.time()
    save_job(job)
    dump = json.dumps({'msg': 'Error', 'data': str(exc)})
    console_key = 'retools:jobconsole:%s' % job.job_id
    current = job.redis.get(console_key)
    if current is not None:
        data = current + str(exc)
    else:
        data = str(exc)
    pl.set(console_key, data)
    pl.srem('retools:started', job.job_id)
    pl.delete('retools:jobpid:%s' % str(os.getpid()))
    pl.sadd('retools:queue:failures', job.job_id)
    pl.lpush('retools:result:%s' % job.job_id, dump)
    pl.sadd('retools:consoles', job.job_id)
    pl.expire('retools:result:%s', 3600)
    nodes = os.environ.get('MARTEAU_NODES')
    if nodes is not None:
        nodes = nodes.split(',')
        for name in nodes:
            data = job.redis.get('retools:node:%s' % name)
            node = Node(**json.loads(data))
            node.status = 'idle'
            names = job.redis.smembers('retools:nodes')
            if node.name not in names:
                job.redis.sadd('retools:nodes', node.name)
            job.redis.set('retools:node:%s' % node.name, node.to_json())

    pl.execute()
def success(job=None, result=None):
    pl = job.redis.pipeline()
    job.metadata['ended'] = time.time()
    save_job(job)
    result = json.dumps({'data': result, 'msg': 'Success'})
    pl.srem('retools:started', job.job_id)
    pl.delete('retools:jobpid:%s' % str(os.getpid()))
    pl.sadd('retools:queue:successes', job.job_id)
    pl.lpush('retools:result:%s' % job.job_id, result)
    pl.expire('retools:result:%s', 3600)
    pl.sadd('retools:consoles', job.job_id)
    nodes = os.environ.get('MARTEAU_NODES')
    if nodes is not None:
        nodes = nodes.split(',')
        for name in nodes:
            data = job.redis.get('retools:node:%s' % name)
            if data is None:
                continue
            node = Node(**json.loads(data))
            node.status = 'idle'
            names = job.redis.smembers('retools:nodes')
            if node.name not in names:
                job.redis.sadd('retools:nodes', node.name)
            job.redis.set('retools:node:%s' % node.name, node.to_json())

    pl.execute()
示例#3
0
def success(job=None, result=None):
    pl = job.redis.pipeline()
    job.metadata['ended'] = time.time()
    save_job(job)
    result = json.dumps({'data': result, 'msg': 'Success'})
    pl.srem('retools:started', job.job_id)
    pl.delete('retools:jobpid:%s' % str(os.getpid()))
    pl.sadd('retools:queue:successes', job.job_id)
    pl.lpush('retools:result:%s' % job.job_id, result)
    pl.expire('retools:result:%s', 3600)
    pl.sadd('retools:consoles', job.job_id)
    nodes = os.environ.get('MARTEAU_NODES')
    if nodes is not None:
        nodes = nodes.split(',')
        for name in nodes:
            data = job.redis.get('retools:node:%s' % name)
            if data is None:
                continue
            node = Node(**json.loads(data))
            node.status = 'idle'
            names = job.redis.smembers('retools:nodes')
            if node.name not in names:
                job.redis.sadd('retools:nodes', node.name)
            job.redis.set('retools:node:%s' % node.name, node.to_json())

    pl.execute()
    def get_nodes(self, check_available=False):
        names = self._conn.smembers('retools:nodes')
        nodes = []

        for name in sorted(names):
            node = self._conn.get('retools:node:%s' % name)
            node = Node(**json.loads(node))
            if check_available and (node.status != 'idle' or not node.enabled):
                continue
            nodes.append(node)

        return nodes
def failure(job=None, exc=None):
    # depending on the failure we might requeue it with a
    # delay, using job.enqueue()
    pl = job.redis.pipeline()
    job.metadata['ended'] = time.time()
    save_job(job)
    dump = json.dumps({'msg': 'Error', 'data': str(exc)})
    console_key = 'retools:jobconsole:%s' % job.job_id
    current = job.redis.get(console_key)
    if current is not None:
        data = current + str(exc) + '\n'

    else:
        data = str(exc) + '\n'
    pl.set(console_key, data)
    pl.srem('retools:started', job.job_id)
    pl.delete('retools:jobpid:%s' % str(os.getpid()))
    pl.sadd('retools:queue:failures', job.job_id)
    pl.lpush('retools:result:%s' % job.job_id, dump)
    pl.sadd('retools:consoles', job.job_id)
    pl.expire('retools:result:%s', 3600)
    nodes = os.environ.get('MARTEAU_NODES')
    if nodes is not None:
        nodes = nodes.split(',')
        for name in nodes:
            data = job.redis.get('retools:node:%s' % name)
            if data is None:
                continue
            node = Node(**json.loads(data))
            node.status = 'idle'
            names = job.redis.smembers('retools:nodes')
            if node.name not in names:
                job.redis.sadd('retools:nodes', node.name)
            job.redis.set('retools:node:%s' % node.name, node.to_json())

    if 'enough free nodes' in str(exc):
        job.enqueue()

    pl.execute()
def add_node(request):
    """Adds a new into Marteau"""
    check_auth(request)
    owner = request.user

    form = Form(request, schema=NodeSchema)

    if not form.validate():
        request.session.flash("Bad node name")
        return HTTPFound(location='/nodes')

    node_name = form.data.get('name')
    node = Node(name=node_name, owner=owner)
    queue = request.registry['queue']
    queue.save_node(node)
    return HTTPFound(location='/nodes')
示例#7
0
    def create_nodes(self,
                     image_id,
                     count,
                     instance_type='t1.micro',
                     security_groups=['marteau'],
                     key_name='tarek'):

        min_count = max_count = count
        reservation = self.conn.run_instances(image_id=image_id,
                                              instance_type=instance_type,
                                              security_groups=security_groups,
                                              key_name=key_name,
                                              min_count=count,
                                              max_count=count)

        for instance in reservation.instances:
            while instance.state != 'running':
                time.sleep(5)
                instance.update()

        return [
            Node(name=instance.public_dns_name, id=instance.id)
            for instance in reservation.instances
        ]
    def test_attr(self):

        node = Node(name='ok', a=1, b=2)
        self.assertEqual(node.a, 1)
 def get_node(self, name):
     data = self._conn.get('retools:node:%s' % name)
     return Node(**json.loads(data))