def f(session_key, txid):
"""
Accepts a transaction id, and returns a list of
updated nodes from input transaction_id to latest
transaction_id. transaction dict.
FIXME: As an optimization, the changes could be
accumulated in a single set up until the point
that someone got a new txid. Then we could avoid
having a long list of one-element transactions, and
instead only create db version intervals on the intervals
that we know people could possibly refer from. If that
makes sense.
Arguments:
txid -- transaction id (opaque)
Returns:
session_key -- unique session key
nodes -- list of updated node_ids from trx_id to
latest transaction id
"""
trans = self.transactions[what]
current_txid = time.time()
if session_key != self.transactions['session_key']:
return generic.http_response(410, 'Invalid session_key')
txid = float(txid)
if 'poll' in request.args:
# we'll poll if we have no changes
if txid >= max(trans.keys()):
semaphore = '%s-changes' % (what)
utility.wait(semaphore)
if txid < min(trans.keys()):
return generic.http_response(410, 'Expired transaction id')
retval = set([])
for x in [trans[tx] for tx in trans.keys() if tx > txid]:
retval = retval.union(x)
return generic.http_response(
200, 'Updated %s' % what.title(),
**{"transaction": {'session_key': session_key,
'txid': '%.6f' % current_txid},
what: list(retval)})
def create():
old_fact = None
# if we are creating with the same host_id and key, then we'll just update
# fields = api._model_get_columns(object_type)
api = api_from_models()
data = flask.request.json
model_object = None
if 'node_id' in data and 'key' in data:
old_fact = api._model_get_first_by_query(
object_type, 'node_id=%d and key="%s"' % (int(data['node_id']),
data['key']))
if old_fact:
model_object = api._model_update_by_id(
object_type, old_fact['id'], data)
# send update notification
generic._notify(model_object, object_type, old_fact['id'])
else:
try:
model_object = api._model_create(object_type, data)
except KeyError as e:
# missing required field
return generic.http_badrequest(msg=str(e))
generic._notify(model_object, object_type, model_object['id'])
href = flask.request.base_url + str(model_object['id'])
return generic.http_response(201, '%s Created' %
singular_object_type.capitalize(), ref=href,
**{singular_object_type: model_object})
def tasks_blocking_by_node_id(node_id):
api = api_from_models()
# README(shep): Using last_checkin attr for agent-health
timestamp = int(time.time())
args = {'node_id': node_id, 'key': 'last_checkin', 'value': timestamp}
try:
r = api.attr_create(args)
except exceptions.IdNotFound:
message = 'Node %s not found.' % args['node_id']
return generic.http_notfound(msg=message)
except exceptions.IdInvalid:
return generic.http_badrequest()
#DB does not hit updater, so we need to notify
generic._update_transaction_id('nodes', id_list=[node_id])
generic._update_transaction_id('attrs', id_list=[r['id']])
while True:
task = api.task_get_first_by_query("node_id=%d and state='pending'" %
int(node_id))
if task is None:
semaphore = 'task-for-%s' % node_id
flask.current_app.logger.debug('waiting on %s' % semaphore)
if not utility.wait(semaphore):
flask.current_app.logger.error("ERROR ON WAIT")
# utility.clear(semaphore)
return generic.http_notfound(msg='no task found')
else:
flask.current_app.logger.error("SUCCESS ON WAIT")
else:
# utility.clear(semaphore)
return generic.http_response(task=task)
def tasks_by_node_id(node_id):
api = api_from_models()
# Display only tasks with state=pending
task = api.task_get_first_by_query("node_id=%d and state='pending'" %
int(node_id))
if not task:
return generic.http_notfound()
else:
resp = generic.http_response(task=task)
task['state'] = 'delivered'
api._model_update_by_id('tasks', task['id'], task)
return resp
def get_ipaddress(host_name):
log = logging.getLogger('.'.join((__name__, 'get_ipaddress')))
log.info('Request received.get_ipaddress')
try:
tokenId = request.cookies.get("iPlanetDirectoryPro")
ori = ool_rm_if.ool_rm_if()
ori.set_auth(tokenId)
nic_info = ori.get_nic_traffic_info(host_name, "C-Plane")
if -1 != nic_info[0]:
return generic.http_response(200, 'success', **{'host_name': host_name, 'ip_address':nic_info[1][0]['ip_address']})
else:
log.error('not found.')
except Exception as e:
log.error('[FATAL]: %s' % str(e))
log.info('Response complete.get_ipaddress')
return generic.http_badrequest(msg='error.')
def create():
old_fact = None
# if we are creating with the same host_id and key, then we'll just update
# fields = api._model_get_columns(object_type)
api = api_from_models()
data = flask.request.json
try:
node_id = data['node_id']
key = data['key']
except TypeError:
return generic.http_badrequest('node_id and key are required.')
except KeyError:
pass
else:
query = 'node_id=%d and key="%s"' % (int(node_id), key)
old_fact = api._model_get_first_by_query(object_type, query)
if old_fact:
return modify_fact(old_fact['id'])
# here, if the fact is a fact on a container,
# we need to solve for fact application on all
# child nodes. <eek>
#
# FIXME(rp): so we'll punt for now, and just refuse fact
# creates on containers.
children = api._model_query('nodes',
'facts.parent_id = %s' % data['node_id'])
if len(children) > 0:
return generic.http_response(403,
msg='cannot set fact on containers',
friendly='oopsie')
constraints = ['facts.%s = "%s"' %
(data['key'],
data['value'])]
return generic.http_solver_request(
data['node_id'], constraints, api=api,
result={'fact': {'id': -1,
'node_id': data['node_id'],
'key': data['key'],
'value': data['value']}})
def root_updates():
"""Returns the latest transaction information from the
in-memory transaction dict. Realize that this is only
accurate at the time of the request. So clearly, one
should call this BEFORE serializing stuffs.
Arguments:
None
Returns:
json object containing: the unique session key,
the latest transaction id
"""
session_key = self.transactions['session_key']
txid = time.time()
return generic.http_response(
transaction={'session_key': session_key,
'txid': '%.6f' % txid})
def whoami():
api = api_from_models()
body = flask.request.json
if body is None or (not 'hostname' in body):
return generic.http_badrequest(
msg="'hostname' not found in json object")
hostname = body['hostname']
nodes = api._model_query(
'nodes',
'name = "%s"' % hostname)
node = None
if len(nodes) == 0:
# register a new node
node = api._model_create('nodes', {'name': hostname})
api._model_create('facts',
{'node_id': node['id'],
'key': 'backends',
'value': ['node', 'agent']})
api._model_create('attrs',
{'node_id': node['id'],
'key': 'converged',
'value': True})
if hostname == socket.gethostname():
api._model_create('facts',
{'node_id': node['id'],
'key': 'parent_id',
'value': 3})
api._model_create('attrs',
{'node_id': node['id'],
'key': 'server-agent',
'value': True})
else:
unprovisioned_id = unprovisioned_container()['id']
api._model_create('facts',
{'node_id': node['id'],
'key': 'parent_id',
'value': unprovisioned_id})
node = api._model_get_by_id('nodes', node['id'])
else:
node = nodes[0]
return generic.http_response(200, 'success',
**{'node': node})
def f(session_key, txid):
"""
Accepts a transaction id, and returns a list of
updated nodes from input transaction_id to latest
transaction_id. transaction dict.
FIXME: As an optimization, the changes could be
accumulated in a single set up until the point
that someone got a new txid. Then we could avoid
having a long list of one-element transactions, and
instead only create db version intervals on the intervals
that we know people could possibly refer from. If that
makes sense.
Arguments:
txid -- transaction id (opaque)
Returns:
session_key -- unique session key
nodes -- list of updated node_ids from trx_id to
latest transaction id
"""
trans = self.transactions[what]
current_txid = time.time()
if session_key != self.transactions['session_key']:
return generic.http_response(410, 'Invalid session_key')
txid = float(txid)
if 'poll' in request.args:
# we'll poll if we have no changes
if txid >= max(trans.keys()):
semaphore = '%s-changes' % (what)
utility.wait(semaphore)
if txid < min(trans.keys()):
return generic.http_response(410, 'Expired transaction id')
retval = set()
for x in (trans[tx] for tx in trans.keys() if tx > txid):
retval.update(x)
#tenant start
#target_tenant = 'test2-tenant' #Temporary
#log("start")
target_tenant = request.headers.get('tenant')
if what == 'nodes' and not generic.is_adminTenant(target_tenant):
try:
node_objects = get_objectlist_not_reserved_tenant(target_tenant)
remove_ids = set()
for node_id in retval:
for obj in node_objects:
#self._logger.debug('obj = %s' % obj)
if int(node_id) == obj['id']:
#self._logger.debug('obj = %s' % obj)
remove_ids.add(node_id)
break
retval.difference_update(remove_ids)
except Invalid_Key:
retval = set()
#log("end")
#tenant end
return generic.http_response(
200, 'Updated %s' % what.title(),
**{"transaction": {'session_key': session_key,
'txid': '%.6f' % current_txid},
what: list(retval)})
def whoami():
log = logging.getLogger('.'.join((__name__, 'whoami')))
log.info('Request received.')
api = api_from_models()
body = flask.request.json
message = 'Node ID or hostname required.'
try:
#if id not supplied assume it is a new agent
node_id = body['node_id']
except TypeError:
return generic.http_badrequest(msg=message)
except KeyError:
try:
hostname = body['hostname']
except KeyError:
return generic.http_badrequest(msg=message)
else:
node = _whoami_backwards_compatibility(api, hostname)
if node is None:
node = api._model_create('nodes', {'name': hostname})
api._model_create('facts',
{'node_id': node['id'],
'key': 'backends',
'value': ['node', 'agent']})
api._model_create('attrs',
{'node_id': node['id'],
'key': 'converged',
'value': True})
api._model_create('attrs',
{'node_id': node['id'],
'key': 'registered',
'value': False})
return generic.http_response(200, 'Node ID assigned.',
node_id=node['id'])
log.info('Node id %s received.' % node_id)
try:
node = api._model_get_by_id('nodes', node_id)
except exceptions.IdNotFound:
message = 'Node %s not found.' % node['node_id']
return generic.http_notfound(msg=message)
except exceptions.IdInvalid:
return generic.http_badrequest('Node ID must be an integer.')
if not node['attrs']['registered']:
# register a new node
log.info('Registering %s' % node_id)
try:
hostidfile = flask.current_app.config['hostidfile']
except KeyError:
log.error('hostidfile not set in config.')
return generic.http_response(500, 'hostidfile not set on server.')
reg_file = '.'.join((hostidfile, 'registering'))
try:
with open(reg_file) as f:
server_id = f.read().strip()
except IOError:
log.error('Unable to read server ID from %s.' % reg_file)
server_id = None
try:
server_node = api._model_get_by_id('nodes', server_id)
except (exceptions.IdInvalid, exceptions.IdNotFound):
#log this as an error and assume agent not on server
log.error('Server ID from in %s is invalid.' % reg_file)
server_node = {'id': None}
if server_node['id'] == node['id']:
api._model_create('facts',
{'node_id': node['id'],
'key': 'parent_id',
'value': 3})
api._model_create('attrs',
{'node_id': node['id'],
'key': 'server-agent',
'value': True})
else:
unprovisioned_id = unprovisioned_container()['id']
api._model_create('facts',
{'node_id': node['id'],
'key': 'parent_id',
'value': unprovisioned_id})
#update registered attr to True
attr_query = "node_id=%d and key='registered'" % node['id']
reg_attr = api.attr_get_first_by_query(attr_query)
reg_attr['value'] = True
api._model_update_by_id('attrs', reg_attr['id'], reg_attr)
node = api._model_get_by_id('nodes', node['id'])
log.info('Registration complete for %s' % node_id)
return generic.http_response(200, 'success', **{'node': node})
def task_log(task_id):
"""
Tail a logfile on a client. Given a task id, this asks
the client that ran it grab the last 1k of the logs from
that task and push them at the server on an ephemeral port.
This gets returned as 'log' in the return json blob.
"""
api = api_from_models()
try:
task = api._model_get_by_id('tasks', task_id)
except exceptions.IdNotFound:
return generic.http_notfound(msg='Task %s not found' % task_id)
watching = flask.request.args.get('watch', False) is not False
offset_raw = flask.request.args.get('offset', '1024')
offset = {}
try:
offset['length'] = int(offset_raw)
except ValueError:
return generic.http_badrequest(msg='Offset must be an integer.')
if offset_raw.startswith('+'):
offset['position'] = 'start'
else:
offset['position'] = 'end'
s = gevent.socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind(('', 0))
s.listen(1)
addr, port = s.getsockname()
if addr == '0.0.0.0':
# we need something more specific. This is
# pretty naive, but works. If we could get the
# flask fd, we could getsockname on that side
# and see what we requested on. that would
# likely be a better guess.
# generate a list of all interfaces with
# non-loopback ipv4 addresses.
addr = None
addrs = {}
for iface in netifaces.interfaces():
if netifaces.AF_INET in netifaces.ifaddresses(iface):
for ablock in netifaces.ifaddresses(iface)[netifaces.AF_INET]:
if not iface in addrs:
addrs[iface] = []
if not ablock['addr'].startswith('127'):
addrs[iface].append(ablock['addr'])
if iface in addrs and len(addrs[iface]) == 0:
addrs.pop(iface)
if len(addrs) == 0:
s.close()
return generic.http_badrequest(msg='cannot determine interface')
# try least-to-most interesting
for iface in ['en1', 'en0', 'eth1', 'eth0']:
if iface in addrs:
addr = addrs[iface][0]
# just grab the first
if not addr:
addr = addrs[addrs.keys().pop(0)][0]
client_action = 'logfile.watch' if watching else 'logfile.tail'
payload = {'node_id': task['node_id'],
'action': client_action,
'payload': {'task_id': task['id'],
'dest_ip': addr,
'dest_port': port,
'offset': offset}}
if watching:
payload['payload']['timeout'] = 30
new_task = api._model_create('tasks', payload)
# this should really be done by the data model. <sigh>
task_semaphore = 'task-for-%s' % task['node_id']
flask.current_app.logger.debug('notifying event %s' %
task_semaphore)
utility.notify(task_semaphore)
# # force the wake
# utility.sleep(0.1)
# now wait for a few seconds to see if we get the
# connection
s.settimeout(10)
try:
conn, addr = s.accept()
except socket.timeout:
flask.current_app.logger.error('Error waiting for '
'client connect on log tail')
s.close()
api._model_update_by_id('tasks', new_task['id'],
{'state': 'cancelled'})
return generic.http_notfound(msg='cannot fetch logs')
if watching:
watch = str(uuid.uuid1())
watched_tasks[watch] = {
'socket': conn,
'time': time.time(),
'task_id': new_task['id'],
'notifier': gevent.event.Event(),
'accept_socket': s,
'event': gevent.spawn(
lambda: _serve_connection(
api, watch))}
return generic.http_response(200, request=watch)
# otherwise, just tail
data = _generate_data(conn, s)
return flask.Response(data, mimetype='text/plain')
def create():
old_fact = None
plan = None
# if we are creating with the same host_id and key, then we'll just update
# fields = api._model_get_columns(object_type)
api = api_from_models()
data = flask.request.json
#logging.debug('facts_please.py:create() data = %s' % data)
try:
node_id = data['node_id']
key = data['key']
except TypeError:
return generic.http_badrequest('node_id and key are required.')
except KeyError:
pass
else:
query = 'node_id=%d and key="%s"' % (int(node_id), key)
old_fact = api._model_get_first_by_query(object_type, query)
#
try:
if (key == 'backends' and 'sdn' not in data['value']):
raise ORIGINAL
if (key == 'parent_id'):
node = api._model_get_by_id('nodes', data['node_id'])
parent = api._model_get_by_id('nodes', data['value'])
if ('sdn' not in node['facts'].get('backends', [])):
if ('sdn' not in parent['facts'].get('backends', {})):
raise ORIGINAL
else:
return generic.http_badrequest('bad parent container')
else:
if ('sdn' not in parent['facts'].get('backends', {})):
return generic.http_badrequest('bad parent container')
# New
# sdn_s
cookie_data = flask.request.headers.get('Cookie')
tokenId = ''
cookie_split=cookie_data.split(';')
for cookie in cookie_split:
index = cookie.find(TOKENKEYWORD)
if index != -1:
tokenId = cookie[index + len(TOKENKEYWORD):]
break
if 0 == len(tokenId):
logging.debug('facts_please.py:create() not find tokenID')
return generic.http_badrequest(msg='not find tokenID')
# logging.debug('TokenID=%s' % tokenId)
# logging.debug('User-Agent=%s' % flask.request.headers.get('User-Agent'))
# logging.debug('tenant=%s' % flask.request.headers.get('tenant'))
# sdn_e
plan = [{'primitive': 'node.set_parent', 'ns': {'parent': data['value']}, 'timeout': 30},
{'primitive': 'node.add_backend', 'ns': {'backend': 'nova'}},
# sdn_s
{'primitive': 'nova.set_sdnbox', 'ns': {'backend': 'sdn','tokenid': tokenId}}]
# sdn_e
if old_fact:
model_object = api._model_get_by_id('facts', old_fact['id'])
if not model_object:
return generic.http_notfound()
node_id = model_object['node_id']
# FIXME: TYPECASTING WHEN WE HAVE FACT TYPES
constraints = ['facts.%s = "%s"' %
(model_object['key'],
data['value'])]
return generic.http_solver_request(
node_id, constraints, api=api,
result={'fact': {'id': model_object['id'],
'node_id': node_id,
'key': model_object['key'],
'value': data['value']}}, plan=plan)
# New
if old_fact:
model_object = api._model_update_by_id(
object_type, old_fact['id'], data)
# send update notification
generic._notify(model_object, object_type, old_fact['id'])
#shellscript = node['attrs'].get('opencenter_sdn_sh', [])
#subprocess.call(shellscript, shell=True)
else:
try:
model_object = api._model_create(object_type, data)
except KeyError as e:
# missing required field
return generic.http_badrequest(msg=str(e))
generic._notify(model_object, object_type, model_object['id'])
href = flask.request.base_url + str(model_object['id'])
return generic.http_response(201, '%s Created' %
singular_object_type.capitalize(), ref=href,
**{singular_object_type: model_object})
except ORIGINAL:
pass
# logging.debug('facts_please.py:create() in ORIGINAL')
#
if old_fact:
logging.debug('facts_please.py:create() goto modify_fact')
return modify_fact(old_fact['id'])
# here, if the fact is a fact on a container,
# we need to solve for fact application on all
# child nodes. <eek>
#
# FIXME(rp): so we'll punt for now, and just refuse fact
# creates on containers.
children = api._model_query('nodes',
'facts.parent_id = %s' % data['node_id'])
if len(children) > 0:
return generic.http_response(403,
msg='cannot set fact on containers',
friendly='oopsie')
constraints = ['facts.%s = "%s"' %
(data['key'],
data['value'])]
return generic.http_solver_request(
data['node_id'], constraints, api=api,
result={'fact': {'id': -1,
'node_id': data['node_id'],
'key': data['key'],
'value': data['value']}}, plan=plan)
def task_log(task_id):
"""
Tail a logfile on a client. Given a task id, this asks
the client that ran it grab the last 1k of the logs from
that task and push them at the server on an ephemeral port.
This gets returned as 'log' in the return json blob.
"""
api = api_from_models()
try:
task = api._model_get_by_id("tasks", task_id)
except exceptions.IdNotFound:
return generic.http_notfound(msg="Task %s not found" % task_id)
watching = flask.request.args.get("watch", False) is not False
offset = flask.request.args.get("offset", 1024)
try:
offset = int(offset)
except ValueError:
pass
if not isinstance(offset, int) or offset < 0:
message = "Offset must be a non-negative integer"
return generic.http_badrequest(msg=message)
s = gevent.socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind(("", 0))
s.listen(1)
addr, port = s.getsockname()
if addr == "0.0.0.0":
# we need something more specific. This is
# pretty naive, but works. If we could get the
# flask fd, we could getsockname on that side
# and see what we requested on. that would
# likely be a better guess.
# generate a list of all interfaces with
# non-loopback ipv4 addresses.
addr = None
addrs = {}
for iface in netifaces.interfaces():
if netifaces.AF_INET in netifaces.ifaddresses(iface):
for ablock in netifaces.ifaddresses(iface)[netifaces.AF_INET]:
if not iface in addrs:
addrs[iface] = []
if not ablock["addr"].startswith("127"):
addrs[iface].append(ablock["addr"])
if iface in addrs and len(addrs[iface]) == 0:
addrs.pop(iface)
if len(addrs) == 0:
s.close()
return generic.http_badrequest(msg="cannot determine interface")
# try least-to-most interesting
for iface in ["en1", "en0", "eth1", "eth0"]:
if iface in addrs:
addr = addrs[iface][0]
# just grab the first
if not addr:
addr = addrs[addrs.keys().pop(0)][0]
client_action = "logfile.watch" if watching else "logfile.tail"
payload = {
"node_id": task["node_id"],
"action": client_action,
"payload": {"task_id": task["id"], "dest_ip": addr, "dest_port": port, "offset": offset},
}
if watching:
payload["payload"]["timeout"] = 30
new_task = api._model_create("tasks", payload)
# this should really be done by the data model. <sigh>
task_semaphore = "task-for-%s" % task["node_id"]
flask.current_app.logger.debug("notifying event %s" % task_semaphore)
utility.notify(task_semaphore)
# force the wake
utility.sleep(0.1)
# now wait for a few seconds to see if we get the
# connection
s.settimeout(10)
try:
conn, addr = s.accept()
except socket.timeout:
flask.current_app.logger.error("Error waiting for " "client connect on log tail")
s.close()
api._model_update_by_id("tasks", new_task["id"], {"state": "cancelled"})
return generic.http_notfound(msg="cannot fetch logs")
if watching:
watch = str(uuid.uuid1())
watched_tasks[watch] = {
"socket": conn,
"time": time.time(),
"task_id": new_task["id"],
"notifier": gevent.event.Event(),
"accept_socket": s,
"event": gevent.spawn(lambda: _serve_connection(api, watch)),
}
return generic.http_response(200, request=watch)
# otherwise, just tail
data = _generate_data(conn, s)
return flask.Response(data, mimetype="text/plain")