Ejemplo n.º 1
0
def tasks_blocking_by_node_id(node_id):
    api = api_from_models()

    # README(shep): Using last_checkin attr for agent-health
    timestamp = int(time.time())
    args = {'node_id': node_id, 'key': 'last_checkin', 'value': timestamp}
    try:
        r = api.attr_create(args)
    except exceptions.IdNotFound:
        message = 'Node %s not found.' % args['node_id']
        return generic.http_notfound(msg=message)
    except exceptions.IdInvalid:
        return generic.http_badrequest()
    #DB does not hit updater, so we need to notify
    generic._update_transaction_id('nodes', id_list=[node_id])
    generic._update_transaction_id('attrs', id_list=[r['id']])

    while True:
        task = api.task_get_first_by_query("node_id=%d and state='pending'" %
                                           int(node_id))
        if task is None:
            semaphore = 'task-for-%s' % node_id
            flask.current_app.logger.debug('waiting on %s' % semaphore)
            if not utility.wait(semaphore):
                flask.current_app.logger.error("ERROR ON WAIT")
                # utility.clear(semaphore)
                return generic.http_notfound(msg='no task found')
            else:
                flask.current_app.logger.error("SUCCESS ON WAIT")
        else:
            # utility.clear(semaphore)
            return generic.http_response(task=task)
Ejemplo n.º 2
0
def task_log_tail(task_id, transaction):
    if not transaction in watched_tasks:
        flask.current_app.logger.error(
            '%s not in watched tasks: %s' % (transaction,
                                             watched_tasks))

        return generic.http_notfound()

    watched_tasks[transaction]['notifier'].set()

    # yield to greenlets
    gevent.sleep(0)

    if not 'generator' in watched_tasks[transaction]:
        # something very wrong here
        flask.current_app.logger.error(
            'no generator in %s: %s' % (
                transaction,
                watched_tasks[transaction]))

        watched_tasks.pop(transaction)
        return generic.http_notfound()

    return flask.Response(watched_tasks[transaction]['generator'],
                          mimetype='text/plain')
Ejemplo n.º 3
0
def modify_fact(object_id):
    """
    Express a fact modification as a list of constraints
    on the linked node.facts, and run the solved
    result
    """
    data = flask.request.json
    api = api_from_models()

    model_object = api._model_get_by_id('facts', object_id)
    if not model_object:
        return generic.http_notfound()

    node_id = model_object['node_id']

    # FIXME: TYPECASTING WHEN WE HAVE FACT TYPES
    constraints = ['facts.%s = "%s"' %
                   (model_object['key'],
                    data['value'])]

    return generic.http_solver_request(
        node_id, constraints, api=api,
        result={'fact': {'id': model_object['id'],
                         'node_id': node_id,
                         'key': model_object['key'],
                         'value': data['value']}})
Ejemplo n.º 4
0
def tasks_blocking_by_node_id(node_id):
    api = api_from_models()

    # README(shep): Using last_checkin attr for agent-health
    timestamp = int(time.time())
    args = {'node_id': node_id, 'key': 'last_checkin', 'value': timestamp}
    r = api.attr_create(args)
    #DB does not hit updater, so we need to notify
    generic._update_transaction_id('nodes', id_list=[node_id])
    generic._update_transaction_id('attrs', id_list=[r['id']])
    task = api.task_get_first_by_query("node_id=%d and state='pending'" %
                                       int(node_id))

    # README(shep): moving this out of a while loop, to let agent-health work
    semaphore = 'task-for-%s' % node_id
    flask.current_app.logger.debug('waiting on %s' % semaphore)
    utility.wait(semaphore)
    task = api.task_get_first_by_query("node_id=%d and state='pending'" %
                                       int(node_id))
    if task:
        utility.clear(semaphore)
        result = flask.jsonify({'task': task})
    else:
        result = generic.http_notfound(msg='no task found')
    return result
Ejemplo n.º 5
0
def execute_adventure(adventure_id):
    data = flask.request.json

    if not 'node' in data:
        return generic.http_badrequest(msg='node not specified')

    api = api_from_models()
    try:
        adventure = api._model_get_by_id('adventures', int(adventure_id))
    except exceptions.IdNotFound:
        message = 'Not Found: Adventure %s' % adventure_id
        return generic.http_notfound(msg=message)

    try:
        return generic.http_solver_request(data['node'], [],
                                           api=api, plan=adventure['dsl'])
    except exceptions.IdNotFound:
        #Can IdNotFound be raised for any other reason?
        return generic.http_notfound(msg='Not Found: Node %s' % data['node'])
Ejemplo n.º 6
0
def tasks_by_node_id(node_id):
    api = api_from_models()
    # Display only tasks with state=pending
    task = api.task_get_first_by_query("node_id=%d and state='pending'" %
                                       int(node_id))
    if not task:
        return generic.http_notfound()
    else:
        resp = generic.http_response(task=task)
        task['state'] = 'delivered'
        api._model_update_by_id('tasks', task['id'], task)
        return resp
Ejemplo n.º 7
0
def adventures_by_node_id(node_id):
    api = api_from_models()
    node = api.node_get_by_id(node_id)
    if not node:
        return generic.http_notfound()
    else:
        all_adventures = api.adventures_get_all()
        available_adventures = []
        for adventure in all_adventures:
            builder = ast.FilterBuilder(ast.FilterTokenizer(),
                                        adventure['criteria'],
                                        api=api)
            try:
                root_node = builder.build()
                if root_node.eval_node(node, builder.functions):
                    available_adventures.append(adventure)
            except Exception as e:
                flask.current_app.logger.warn(
                    'adv err %s: %s' % (adventure['name'], str(e)))

        # adventures = api.adventures_get_by_node_id(node_id)
        resp = flask.jsonify({'adventures': available_adventures})
    return resp
Ejemplo n.º 8
0
def whoami():
    log = logging.getLogger('.'.join((__name__, 'whoami')))
    log.info('Request received.')
    api = api_from_models()
    body = flask.request.json
    message = 'Node ID or hostname required.'
    try:
        #if id not supplied assume it is a new agent
        node_id = body['node_id']
    except TypeError:
        return generic.http_badrequest(msg=message)
    except KeyError:
        try:
            hostname = body['hostname']
        except KeyError:
            return generic.http_badrequest(msg=message)
        else:
            node = _whoami_backwards_compatibility(api, hostname)
            if node is None:
                node = api._model_create('nodes', {'name': hostname})
                api._model_create('facts',
                                  {'node_id': node['id'],
                                   'key': 'backends',
                                   'value': ['node', 'agent']})
                api._model_create('attrs',
                                  {'node_id': node['id'],
                                   'key': 'converged',
                                   'value': True})
                api._model_create('attrs',
                                  {'node_id': node['id'],
                                   'key': 'registered',
                                   'value': False})
            return generic.http_response(200, 'Node ID assigned.',
                                         node_id=node['id'])
    log.info('Node id %s received.' % node_id)
    try:
        node = api._model_get_by_id('nodes', node_id)
    except exceptions.IdNotFound:
        message = 'Node %s not found.' % node['node_id']
        return generic.http_notfound(msg=message)
    except exceptions.IdInvalid:
        return generic.http_badrequest('Node ID must be an integer.')

    if not node['attrs']['registered']:
        # register a new node
        log.info('Registering %s' % node_id)
        try:
            hostidfile = flask.current_app.config['hostidfile']
        except KeyError:
            log.error('hostidfile not set in config.')
            return generic.http_response(500, 'hostidfile not set on server.')
        reg_file = '.'.join((hostidfile, 'registering'))
        try:
            with open(reg_file) as f:
                server_id = f.read().strip()
        except IOError:
            log.error('Unable to read server ID from %s.' % reg_file)
            server_id = None
        try:
            server_node = api._model_get_by_id('nodes', server_id)
        except (exceptions.IdInvalid, exceptions.IdNotFound):
            #log this as an error and assume agent not on server
            log.error('Server ID from in %s is invalid.' % reg_file)
            server_node = {'id': None}
        if server_node['id'] == node['id']:
            api._model_create('facts',
                              {'node_id': node['id'],
                               'key': 'parent_id',
                               'value': 3})
            api._model_create('attrs',
                              {'node_id': node['id'],
                              'key': 'server-agent',
                              'value': True})
        else:
            unprovisioned_id = unprovisioned_container()['id']
            api._model_create('facts',
                              {'node_id': node['id'],
                               'key': 'parent_id',
                               'value': unprovisioned_id})
        #update registered attr to True
        attr_query = "node_id=%d and key='registered'" % node['id']
        reg_attr = api.attr_get_first_by_query(attr_query)
        reg_attr['value'] = True
        api._model_update_by_id('attrs', reg_attr['id'], reg_attr)
        node = api._model_get_by_id('nodes', node['id'])
        log.info('Registration complete for %s' % node_id)
    return generic.http_response(200, 'success', **{'node': node})
Ejemplo n.º 9
0
def task_log(task_id):
    """
    Tail a logfile on a client.  Given a task id, this asks
    the client that ran it grab the last 1k of the logs from
    that task and push them at the server on an ephemeral port.

    This gets returned as 'log' in the return json blob.
    """

    api = api_from_models()
    try:
        task = api._model_get_by_id('tasks', task_id)
    except exceptions.IdNotFound:
        return generic.http_notfound(msg='Task %s not found' % task_id)

    watching = flask.request.args.get('watch', False) is not False

    offset_raw = flask.request.args.get('offset', '1024')
    offset = {}
    try:
        offset['length'] = int(offset_raw)
    except ValueError:
        return generic.http_badrequest(msg='Offset must be an integer.')

    if offset_raw.startswith('+'):
        offset['position'] = 'start'
    else:
        offset['position'] = 'end'

    s = gevent.socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    s.bind(('', 0))
    s.listen(1)

    addr, port = s.getsockname()
    if addr == '0.0.0.0':
        # we need something more specific.  This is
        # pretty naive, but works.  If we could get the
        # flask fd, we could getsockname on that side
        # and see what we requested on.  that would
        # likely be a better guess.

        # generate a list of all interfaces with
        # non-loopback ipv4 addresses.
        addr = None

        addrs = {}
        for iface in netifaces.interfaces():
            if netifaces.AF_INET in netifaces.ifaddresses(iface):
                for ablock in netifaces.ifaddresses(iface)[netifaces.AF_INET]:
                    if not iface in addrs:
                        addrs[iface] = []
                    if not ablock['addr'].startswith('127'):
                        addrs[iface].append(ablock['addr'])

                if iface in addrs and len(addrs[iface]) == 0:
                    addrs.pop(iface)

        if len(addrs) == 0:
            s.close()
            return generic.http_badrequest(msg='cannot determine interface')

        # try least-to-most interesting
        for iface in ['en1', 'en0', 'eth1', 'eth0']:
            if iface in addrs:
                addr = addrs[iface][0]

        # just grab the first
        if not addr:
            addr = addrs[addrs.keys().pop(0)][0]

    client_action = 'logfile.watch' if watching else 'logfile.tail'

    payload = {'node_id': task['node_id'],
               'action': client_action,
               'payload': {'task_id': task['id'],
                           'dest_ip': addr,
                           'dest_port': port,
                           'offset': offset}}
    if watching:
        payload['payload']['timeout'] = 30

    new_task = api._model_create('tasks', payload)

    # this should really be done by the data model.  <sigh>
    task_semaphore = 'task-for-%s' % task['node_id']
    flask.current_app.logger.debug('notifying event %s' %
                                   task_semaphore)
    utility.notify(task_semaphore)

    # # force the wake
    # utility.sleep(0.1)

    # now wait for a few seconds to see if we get the
    # connection
    s.settimeout(10)

    try:
        conn, addr = s.accept()
    except socket.timeout:
        flask.current_app.logger.error('Error waiting for '
                                       'client connect on log tail')

        s.close()
        api._model_update_by_id('tasks', new_task['id'],
                                {'state': 'cancelled'})

        return generic.http_notfound(msg='cannot fetch logs')

    if watching:
        watch = str(uuid.uuid1())
        watched_tasks[watch] = {
            'socket': conn,
            'time': time.time(),
            'task_id': new_task['id'],
            'notifier': gevent.event.Event(),
            'accept_socket': s,
            'event': gevent.spawn(
                lambda: _serve_connection(
                    api, watch))}

        return generic.http_response(200, request=watch)

    # otherwise, just tail
    data = _generate_data(conn, s)

    return flask.Response(data, mimetype='text/plain')
Ejemplo n.º 10
0
def create():
    old_fact = None
    plan = None

    # if we are creating with the same host_id and key, then we'll just update
    # fields = api._model_get_columns(object_type)

    api = api_from_models()
    data = flask.request.json

    #logging.debug('facts_please.py:create() data = %s' % data)
    try:
        node_id = data['node_id']
        key = data['key']
    except TypeError:
        return generic.http_badrequest('node_id and key are required.')
    except KeyError:
        pass
    else:
        query = 'node_id=%d and key="%s"' % (int(node_id), key)
        old_fact = api._model_get_first_by_query(object_type, query)

    #
    try:
        if (key == 'backends' and 'sdn' not in data['value']):
            raise ORIGINAL 

        if (key == 'parent_id'):
            node = api._model_get_by_id('nodes', data['node_id'])
            parent = api._model_get_by_id('nodes', data['value'])
            if ('sdn' not in node['facts'].get('backends', [])):
                if ('sdn' not in parent['facts'].get('backends', {})):
                    raise ORIGINAL
                else:
                    return generic.http_badrequest('bad parent container')
            else:
                if ('sdn' not in parent['facts'].get('backends', {})):
                    return generic.http_badrequest('bad parent container')

        # New
        # sdn_s
        cookie_data = flask.request.headers.get('Cookie')

        tokenId = ''
        cookie_split=cookie_data.split(';')
        for cookie in cookie_split:
            index = cookie.find(TOKENKEYWORD)
            if index != -1:
                tokenId = cookie[index + len(TOKENKEYWORD):]
                break

        if 0 == len(tokenId):
            logging.debug('facts_please.py:create() not find tokenID')
            return generic.http_badrequest(msg='not find tokenID')

#        logging.debug('TokenID=%s' % tokenId)
#        logging.debug('User-Agent=%s' % flask.request.headers.get('User-Agent'))
#        logging.debug('tenant=%s' % flask.request.headers.get('tenant'))	
        # sdn_e
        plan = [{'primitive': 'node.set_parent', 'ns': {'parent': data['value']}, 'timeout': 30},
                {'primitive': 'node.add_backend', 'ns': {'backend': 'nova'}},
        # sdn_s
                {'primitive': 'nova.set_sdnbox', 'ns': {'backend': 'sdn','tokenid': tokenId}}]
        # sdn_e

        if old_fact:
            model_object = api._model_get_by_id('facts', old_fact['id'])
            if not model_object:
                return generic.http_notfound()

            node_id = model_object['node_id']

            # FIXME: TYPECASTING WHEN WE HAVE FACT TYPES
            constraints = ['facts.%s = "%s"' %
                          (model_object['key'],
                          data['value'])]

            return generic.http_solver_request(
                    node_id, constraints, api=api,
                    result={'fact': {'id': model_object['id'],
                             'node_id': node_id,
                             'key': model_object['key'],
                             'value': data['value']}}, plan=plan)
        # New

        if old_fact:
            model_object = api._model_update_by_id(
                    object_type, old_fact['id'], data)
            # send update notification
            generic._notify(model_object, object_type, old_fact['id'])
            #shellscript = node['attrs'].get('opencenter_sdn_sh', [])
            #subprocess.call(shellscript, shell=True)
        else:
            try:
                model_object = api._model_create(object_type, data)
            except KeyError as e:
               # missing required field
                return generic.http_badrequest(msg=str(e))

            generic._notify(model_object, object_type, model_object['id'])

        href = flask.request.base_url + str(model_object['id'])
        return generic.http_response(201, '%s Created' %
                             singular_object_type.capitalize(), ref=href,
                             **{singular_object_type: model_object})
    
    except ORIGINAL:
        pass
        # logging.debug('facts_please.py:create() in ORIGINAL')
    #

    if old_fact:
        logging.debug('facts_please.py:create() goto modify_fact')
        return modify_fact(old_fact['id'])

    # here, if the fact is a fact on a container,
    # we need to solve for fact application on all
    # child nodes.  <eek>
    #
    # FIXME(rp): so we'll punt for now, and just refuse fact
    # creates on containers.
    children = api._model_query('nodes',
                                'facts.parent_id = %s' % data['node_id'])
    if len(children) > 0:
        return generic.http_response(403,
                                     msg='cannot set fact on containers',
                                     friendly='oopsie')

    constraints = ['facts.%s = "%s"' %
                   (data['key'],
                    data['value'])]

    return generic.http_solver_request(
        data['node_id'], constraints, api=api,
        result={'fact': {'id': -1,
                         'node_id': data['node_id'],
                         'key': data['key'],
                         'value': data['value']}}, plan=plan)
Ejemplo n.º 11
0
def task_log(task_id):
    """
    Tail a logfile on a client.  Given a task id, this asks
    the client that ran it grab the last 1k of the logs from
    that task and push them at the server on an ephemeral port.

    This gets returned as 'log' in the return json blob.
    """

    api = api_from_models()
    try:
        task = api._model_get_by_id("tasks", task_id)
    except exceptions.IdNotFound:
        return generic.http_notfound(msg="Task %s not found" % task_id)

    watching = flask.request.args.get("watch", False) is not False

    offset = flask.request.args.get("offset", 1024)
    try:
        offset = int(offset)
    except ValueError:
        pass
    if not isinstance(offset, int) or offset < 0:
        message = "Offset must be a non-negative integer"
        return generic.http_badrequest(msg=message)

    s = gevent.socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    s.bind(("", 0))
    s.listen(1)

    addr, port = s.getsockname()
    if addr == "0.0.0.0":
        # we need something more specific.  This is
        # pretty naive, but works.  If we could get the
        # flask fd, we could getsockname on that side
        # and see what we requested on.  that would
        # likely be a better guess.

        # generate a list of all interfaces with
        # non-loopback ipv4 addresses.
        addr = None

        addrs = {}
        for iface in netifaces.interfaces():
            if netifaces.AF_INET in netifaces.ifaddresses(iface):
                for ablock in netifaces.ifaddresses(iface)[netifaces.AF_INET]:
                    if not iface in addrs:
                        addrs[iface] = []
                    if not ablock["addr"].startswith("127"):
                        addrs[iface].append(ablock["addr"])

                if iface in addrs and len(addrs[iface]) == 0:
                    addrs.pop(iface)

        if len(addrs) == 0:
            s.close()
            return generic.http_badrequest(msg="cannot determine interface")

        # try least-to-most interesting
        for iface in ["en1", "en0", "eth1", "eth0"]:
            if iface in addrs:
                addr = addrs[iface][0]

        # just grab the first
        if not addr:
            addr = addrs[addrs.keys().pop(0)][0]

    client_action = "logfile.watch" if watching else "logfile.tail"

    payload = {
        "node_id": task["node_id"],
        "action": client_action,
        "payload": {"task_id": task["id"], "dest_ip": addr, "dest_port": port, "offset": offset},
    }
    if watching:
        payload["payload"]["timeout"] = 30

    new_task = api._model_create("tasks", payload)

    # this should really be done by the data model.  <sigh>
    task_semaphore = "task-for-%s" % task["node_id"]
    flask.current_app.logger.debug("notifying event %s" % task_semaphore)
    utility.notify(task_semaphore)

    # force the wake
    utility.sleep(0.1)

    # now wait for a few seconds to see if we get the
    # connection
    s.settimeout(10)

    try:
        conn, addr = s.accept()
    except socket.timeout:
        flask.current_app.logger.error("Error waiting for " "client connect on log tail")

        s.close()
        api._model_update_by_id("tasks", new_task["id"], {"state": "cancelled"})

        return generic.http_notfound(msg="cannot fetch logs")

    if watching:
        watch = str(uuid.uuid1())
        watched_tasks[watch] = {
            "socket": conn,
            "time": time.time(),
            "task_id": new_task["id"],
            "notifier": gevent.event.Event(),
            "accept_socket": s,
            "event": gevent.spawn(lambda: _serve_connection(api, watch)),
        }

        return generic.http_response(200, request=watch)

    # otherwise, just tail
    data = _generate_data(conn, s)

    return flask.Response(data, mimetype="text/plain")