def run_adventure(adventure_dsl=None, nodes=None):
"""
run an arbitrary adventure on a set of nodes, either by ID or
as an ad-hoc adventure dsl.
There are going to be issues with running ad-hoc adventures including
logging and tracking problems. these should be straightened out.
"""
payload = {}
adv_globals = {}
payload['adventure_dsl'] = adventure_dsl
payload['globals'] = adv_globals
node_list = nodes
api = api_from_models()
# we will no longer expand node lists. At some point
# we either need hints on adventures for whether they are
# targetted at nodes or containers, or whether they
# should be expanded. Or perhaps offer an API call
# to expand node lists. Or something else altogether.
# LOG.debug('node list before expansion: %s' % nodes)
# node_list = expand_nodelist(nodes)
# LOG.debug('node list after expansion: %s' % node_list)
if len(node_list) == 0:
raise ValueError('no nodes specified to run on')
payload['nodes'] = node_list
# find the node with the adventurator plugin
query = "'adventurator' in attrs.opencenter_agent_output_modules"
adventure_nodes = api.nodes_query(query)
# find out how long this should run...
max_time = 0
for step in adventure_dsl:
max_time += step.get('timeout', 30)
if len(adventure_nodes) > 0:
adventure_node = adventure_nodes.pop(0)['id']
task = api.task_create({'action': 'adventurate',
'node_id': adventure_node,
'payload': payload,
'expires': int(time.time() + max_time)})
# FAIL: create a task update -- this should be done
# via the model...
task_semaphore = 'task-for-%s' % adventure_node
notify(task_semaphore)
else:
raise ValueError('no adventurator')
return task
def tasks_blocking_by_node_id(node_id):
api = api_from_models()
# README(shep): Using last_checkin attr for agent-health
timestamp = int(time.time())
args = {'node_id': node_id, 'key': 'last_checkin', 'value': timestamp}
try:
r = api.attr_create(args)
except exceptions.IdNotFound:
message = 'Node %s not found.' % args['node_id']
return generic.http_notfound(msg=message)
except exceptions.IdInvalid:
return generic.http_badrequest()
#DB does not hit updater, so we need to notify
generic._update_transaction_id('nodes', id_list=[node_id])
generic._update_transaction_id('attrs', id_list=[r['id']])
while True:
task = api.task_get_first_by_query("node_id=%d and state='pending'" %
int(node_id))
if task is None:
semaphore = 'task-for-%s' % node_id
flask.current_app.logger.debug('waiting on %s' % semaphore)
if not utility.wait(semaphore):
flask.current_app.logger.error("ERROR ON WAIT")
# utility.clear(semaphore)
return generic.http_notfound(msg='no task found')
else:
flask.current_app.logger.error("SUCCESS ON WAIT")
else:
# utility.clear(semaphore)
return generic.http_response(task=task)
def modify_fact(object_id):
"""
Express a fact modification as a list of constraints
on the linked node.facts, and run the solved
result
"""
data = flask.request.json
api = api_from_models()
model_object = api._model_get_by_id('facts', object_id)
if not model_object:
return generic.http_notfound()
node_id = model_object['node_id']
# FIXME: TYPECASTING WHEN WE HAVE FACT TYPES
constraints = ['facts.%s = "%s"' %
(model_object['key'],
data['value'])]
return generic.http_solver_request(
node_id, constraints, api=api,
result={'fact': {'id': model_object['id'],
'node_id': node_id,
'key': model_object['key'],
'value': data['value']}})
def _clean_tasks():
"""
clean up completed tasks over task_reaping_threshold seconds old.
This is set in the [main] section of the config. The default is
300 seconds (5 min).
This will not run more often than once per minute.
"""
current_time = time.time()
if current_time < task_last_run + 60:
return
api = api_from_models()
reaping_threshold = flask.current_app.config['task_reaping_threshold']
expiration_threshold = current_time - int(reaping_threshold)
expired_tasks = api._model_query(
object_type,
'(state = "done" or state="cancelled") '
'and completed < %d' %
expiration_threshold)
for task in expired_tasks:
api._model_delete_by_id(object_type, task['id'])
def create():
old_fact = None
# if we are creating with the same host_id and key, then we'll just update
# fields = api._model_get_columns(object_type)
api = api_from_models()
data = flask.request.json
model_object = None
if 'node_id' in data and 'key' in data:
old_fact = api._model_get_first_by_query(
object_type, 'node_id=%d and key="%s"' % (int(data['node_id']),
data['key']))
if old_fact:
model_object = api._model_update_by_id(
object_type, old_fact['id'], data)
# send update notification
generic._notify(model_object, object_type, old_fact['id'])
else:
try:
model_object = api._model_create(object_type, data)
except KeyError as e:
# missing required field
return generic.http_badrequest(msg=str(e))
generic._notify(model_object, object_type, model_object['id'])
href = flask.request.base_url + str(model_object['id'])
return generic.http_response(201, '%s Created' %
singular_object_type.capitalize(), ref=href,
**{singular_object_type: model_object})
def setUp(self):
if opencenter.backends.primitive_by_name('test.set_test_fact') is None:
opencenter.backends.load_specific_backend('tests.test',
'TestBackend')
if opencenter.backends.primitive_by_name('test2.add_backend') is None:
opencenter.backends.load_specific_backend('tests.test2',
'Test2Backend')
self._clean_all()
self.interfaces = {}
self.adv = self._stub_node(
'adventurator',
facts={'backends': ['node', 'agent']})
self.container = self._stub_node(
'container',
facts={'backends': ['node', 'container']})
self.node = self._stub_node('node-1')
chef_expr = '(facts.chef_server_uri != None) and ' \
'(facts.chef_server_pem != None)'
# some of our current primitives require this
self.interfaces['chef'] = self._model_create('filters',
name='chef-server',
filter_type='interface',
expr=chef_expr)
self.api = db_api.api_from_models()
self.assertEquals(len(self._model_get_all('tasks')), 0)
def object_by_id(object_type, object_id):
s_obj = singularize(object_type)
api = api_from_models()
if flask.request.method == 'PUT':
# we just updated something, poke any waiters
model_object = api._model_update_by_id(object_type, object_id,
flask.request.json)
_notify(model_object, object_type, object_id)
return http_response(200, '%s Updated' % s_obj.capitalize(),
**{s_obj: model_object})
elif flask.request.method == 'DELETE':
try:
if api._model_delete_by_id(object_type, object_id):
return http_response(200, '%s deleted' % s_obj.capitalize())
_notify(None, object_type, object_id)
except exceptions.IdNotFound:
return http_notfound(msg='not found')
elif flask.request.method == 'GET':
if 'poll' in flask.request.args:
# we're polling
semaphore = '%s-id-%s' % (object_type, object_id)
utility.wait(semaphore)
try:
model_object = api._model_get_by_id(object_type, object_id)
except exceptions.IdNotFound:
return http_notfound(msg='not found')
return http_response(200, 'success', **{s_obj: model_object})
else:
return http_notfound(msg='Unknown method %s' % flask.request.method)
def tasks_blocking_by_node_id(node_id):
api = api_from_models()
# README(shep): Using last_checkin attr for agent-health
timestamp = int(time.time())
args = {'node_id': node_id, 'key': 'last_checkin', 'value': timestamp}
r = api.attr_create(args)
#DB does not hit updater, so we need to notify
generic._update_transaction_id('nodes', id_list=[node_id])
generic._update_transaction_id('attrs', id_list=[r['id']])
task = api.task_get_first_by_query("node_id=%d and state='pending'" %
int(node_id))
# README(shep): moving this out of a while loop, to let agent-health work
semaphore = 'task-for-%s' % node_id
flask.current_app.logger.debug('waiting on %s' % semaphore)
utility.wait(semaphore)
task = api.task_get_first_by_query("node_id=%d and state='pending'" %
int(node_id))
if task:
utility.clear(semaphore)
result = flask.jsonify({'task': task})
else:
result = generic.http_notfound(msg='no task found')
return result
def run_plan():
# this comes in just like an optioned plan. We'll stuff any returned
# args and call it a plan. <rimshot>
#
api = api_from_models()
data = flask.request.json
if not 'node' in data:
return generic.http_badrequest(msg='no node specified')
if not 'plan' in data:
return generic.http_badrequest(msg='no plan specified')
plan = data['plan']
# this is more than a bit awkward
for step in plan:
if 'args' in step:
for arg in step['args']:
if 'value' in step['args'][arg]:
step['ns'][arg] = step['args'][arg]['value']
step.pop('args')
# now our plan is a standard plan. Let's run it
return generic.http_solver_request(data['node'], [], api=api, plan=plan)
def _expand_nodes(nodelist, filter_f=true_f,
api=None, depth=0, detailed=False):
if api is None:
api = api_from_models()
final_nodes = []
nodes = copy.deepcopy(nodelist)
seen = {}
for node in nodes:
if isinstance(node, (int, long)):
node = api.node_get_by_id(node)
if not node['id'] in seen:
seen[node['id']] = {"inspect": False, "level": 0}
elif not seen[node['id']]['inspect']:
#We've already inspected this node, move on.
next
#We're about to inspect the node. Don't do it if we see the node again
seen[node['id']]['inspect'] = False
if filter_f(node):
if detailed:
final_nodes.append(node)
elif node is not None:
final_nodes.append(node['id'])
#We'll add new children provided we're not out of depth
if is_container(node) and (depth == 0
or seen[node['id']]['level'] < depth):
new_nodes = api.nodes_query('facts.parent_id = %s' % node['id'])
for new_n in new_nodes:
if not new_n['id'] in seen:
seen[new_n['id']] = {
"inspect": True,
"level": seen[node['id']]['level'] + 1}
nodes.append(new_n)
return final_nodes
def f(filter_id):
api = api_from_models()
filter_obj = api.filter_get_by_id(filter_id)
full_expr = filter_obj['full_expr']
builder = FilterBuilder(FilterTokenizer(),
'%s: %s' % (what, full_expr))
return jsonify({what: builder.filter()})
def get_direct_children(node_id, api=None):
"""
given a node_id, return a list of all direct child nodes
"""
if api is None:
api = api_from_models()
return [x for x in _expand_nodes([node_id], api=api,
depth=1, detailed=True)
if x['id'] != node_id]
def fully_expand_nodelist(nodelist, api=None):
"""
given a list of nodes (including containers),
generate a fully expanded list of all node_ids in node_list
as well as their descendant nodes
"""
if api is None:
api = api_from_models()
return _expand_nodes(nodelist, api=api)
def expand_nodelist(nodelist, api=None):
"""
given a list of nodes (including containers),
generate a fully expanded list of non-container-y
nodes
"""
if api is None:
api = api_from_models()
return _expand_nodes(nodelist, api=api, filter_f=is_leaf)
def upgrade(migrate_engine):
meta = MetaData(bind=migrate_engine)
api = api_from_models()
for adventure in adventures:
new_adventure = {'name': adventure['name']}
json_path = os.path.join(
os.path.dirname(__file__), adventure['dsl'])
criteria_path = os.path.join(
os.path.dirname(__file__), adventure['criteria'])
new_adventure['dsl'] = json.loads(open(json_path).read())
new_adventure['criteria'] = open(criteria_path).read()
api.adventure_create(new_adventure)
canned_filters = [{'name': 'unprovisioned nodes',
'filter_type': 'node',
'expr': 'backend=\'unprovisioned\''},
{'name': 'chef client nodes',
'filter_type': 'node',
'expr': 'backend=\'chef-client\''},
{'name': 'chef-server',
'filter_type': 'interface',
'expr': 'facts.chef_server_uri != None and '
'facts.chef_server_pem != None'}]
for new_filter in canned_filters:
api._model_create('filters', new_filter)
workspace = api.node_create({'name': 'workspace'})
api._model_create('attrs', {'node_id': workspace['id'],
'key': 'json_schema_version',
'value': 1})
unprov = api.node_create({'name': 'unprovisioned'})
api._model_create('facts', {'node_id': unprov['id'],
'key': 'parent_id',
'value': workspace['id']})
support = api.node_create({'name': 'support'})
api._model_create('facts', {'node_id': support['id'],
'key': 'parent_id',
'value': workspace['id']})
# Add default fact to the default nodes
node_list = [(workspace, "Workspace"),
(unprov, "Available Nodes"),
(support, "Service Nodes")]
for node, display in node_list:
api.fact_create({'node_id': node['id'],
'key': 'backends',
'value': ["container", "node"]})
api.attr_create({'node_id': node['id'],
'key': 'display_name',
'value': display})
api.attr_create({'node_id': node['id'],
'key': 'locked',
'value': True})
def setUp(self):
self.server = self._model_create('nodes', name='opencenter-server')
self.agent = self._model_create('nodes', name='opencenter-client')
for node_id in [self.server['id'], self.agent['id']]:
self._make_facts(node_id,
{'parent_id': 2,
'backends': ['agent', 'node']})
# grab the api so we can roll stuff forward
self.api = db_api.api_from_models()
def get_objectlist_not_reserved_tenant(target_tenant):
api = api_from_models()
# get a nodes of mismatch attr tenant
query = 'attrs.tenant!=null and attrs.tenant!="%s" and attrs.tenant!="any"' % target_tenant
node_objects = api.nodes_query(query)
# get a nodes of server and sdn box
query = '"agent" in facts.backends and attrs.tenant!="any"'
node_obj_agents = api.nodes_query(query)
# get devide list that are assigned to the tenannt from the resouce managers
ori=ool_rm_if.ool_rm_if()
cookie_data = request.headers.get('Cookie')
tokenId = ''
cookie_split=cookie_data.split(';')
for cookie in cookie_split:
index = cookie.find(TOKENKEYWORD)
if index != -1:
tokenId = cookie[index + len(TOKENKEYWORD):]
break
if 0 == len(tokenId):
logging.debug('facts_please.py:create() not find tokenID')
return node_objects
#logging.debug('TokenID=%s' % tokenId)
#logging.debug('User-Agent=%s' % flask.request.headers.get('User-Agent'))
#logging.debug('tenant=%s' % request.headers.get('tenant'))
ori.set_auth(tokenId)
data = ori.get_tenant('', target_tenant)
#logging.debug('data = %s' % data)
# return err
if -1 == data[0]:
logging.debug("ori.get_tenant err data=%s" %(data))
raise Invalid_Key
# set result
result = data[1]
# Add to list server & SDN box that are not reserved by the tenant
for obj in node_obj_agents:
for node in result:
if obj['name'] == node['device_name']:
break
else:
#logging.debug('obj = %s' % obj)
node_objects.append(obj)
return node_objects
def solve_and_run(node_id, constraints, api=None, plan=None):
if api is None:
api = api_from_models()
is_solvable, requires_input, solution_plan = solve_for_node(
node_id, constraints, api=api, plan=plan)
task = None
if is_solvable:
task = run_adventure(adventure_dsl=solution_plan, nodes=[node_id])
return task, is_solvable, requires_input, solution_plan
def tasks_by_node_id(node_id):
api = api_from_models()
# Display only tasks with state=pending
task = api.task_get_first_by_query("node_id=%d and state='pending'" %
int(node_id))
if not task:
return generic.http_notfound()
else:
resp = generic.http_response(task=task)
task['state'] = 'delivered'
api._model_update_by_id('tasks', task['id'], task)
return resp
def task_by_id(object_id):
result = generic.object_by_id(object_type, object_id)
if flask.request.method == "PUT":
api = api_from_models()
task = api.task_get_by_id(object_id)
if "node_id" in task:
flask.current_app.logger.debug("Task: %s" % task)
task_semaphore = "task-for-%s" % task["node_id"]
flask.current_app.logger.debug("notifying event %s" % task_semaphore)
utility.notify(task_semaphore)
return result
def task_by_id(object_id):
result = generic.object_by_id(object_type, object_id)
if flask.request.method == 'PUT':
api = api_from_models()
task = api.task_get_by_id(object_id)
if 'node_id' in task:
flask.current_app.logger.debug('Task: %s' % task)
task_semaphore = 'task-for-%s' % task['node_id']
flask.current_app.logger.debug('notifying event %s' %
task_semaphore)
utility.notify(task_semaphore)
return result
def list_index():
api = api_from_models()
models = api._get_models()
url = flask.request.url
msg = {'url': flask.request.url,
'resources': {}}
for model in models:
msg['resources'][model] = {'url': '%s%s/' % (url, model)}
resp = flask.jsonify(msg)
resp.status_code = 200
return resp
def downgrade(migrate_engine):
meta = MetaData(bind=migrate_engine)
api = api_from_models()
for adventure in adventures:
db_entries = api._model_query('adventures',
'name="%s"' % adventure['name'])
if len(db_entries) == 1:
db_entry = db_entries[0]
criteria_path = os.path.join(
os.path.dirname(__file__), adventure['criteria']['001'])
db_entry['criteria'] = open(criteria_path).read()
api.adventure_update_by_id(db_entry['id'], db_entry)
def _notify(updated_object, object_type, object_id):
semaphore = '%s-id-%s' % (object_type, object_id)
utility.notify(semaphore)
# TODO: Generalize the block of code with a TODO below here.
# if updated_object is not None and object_type in related_notifications:
# for field, entity in related_notifications[object_type].iteritems():
# if field in updated_object and updated_object[field] is not None:
# semaphore = '%s-id-%s' % (entity, updated_object[field])
# utility.notify(semaphore)
# TODO (wilk or rpedde): Use specific notifications for inheritance
if object_type not in ('attrs', 'facts', 'nodes'):
return
try:
node_id = updated_object['node_id']
node = None
except KeyError:
node_id = updated_object['id']
node = updated_object
if object_type != "attrs":
api = api_from_models()
# We're just going to notify every child when containers are updated
if node is None:
try:
node = api._model_get_by_id('nodes', node_id)
except (exceptions.IdNotFound):
return
if 'container' in node['facts'].get('backends', []):
children = utility.get_direct_children(node, api)
for child in children:
semaphore = 'nodes-id-%s' % child['id']
utility.notify(semaphore)
# Update transaction for node and children
id_list = utility.fully_expand_nodelist([node], api)
# TODO(shep): this needs to be better abstracted
# Need a codepath to update transaction for attr modifications
else:
# TODO(shep): this needs to be better abstracted
id_list = [node_id]
_update_transaction_id('nodes', id_list)
def create():
old_fact = None
# if we are creating with the same host_id and key, then we'll just update
# fields = api._model_get_columns(object_type)
api = api_from_models()
data = flask.request.json
try:
node_id = data['node_id']
key = data['key']
except TypeError:
return generic.http_badrequest('node_id and key are required.')
except KeyError:
pass
else:
query = 'node_id=%d and key="%s"' % (int(node_id), key)
old_fact = api._model_get_first_by_query(object_type, query)
if old_fact:
return modify_fact(old_fact['id'])
# here, if the fact is a fact on a container,
# we need to solve for fact application on all
# child nodes. <eek>
#
# FIXME(rp): so we'll punt for now, and just refuse fact
# creates on containers.
children = api._model_query('nodes',
'facts.parent_id = %s' % data['node_id'])
if len(children) > 0:
return generic.http_response(403,
msg='cannot set fact on containers',
friendly='oopsie')
constraints = ['facts.%s = "%s"' %
(data['key'],
data['value'])]
return generic.http_solver_request(
data['node_id'], constraints, api=api,
result={'fact': {'id': -1,
'node_id': data['node_id'],
'key': data['key'],
'value': data['value']}})
def set_restore_list(plan, solution_plan, node_id):
plan_dict = plan[0]
solution_plan_dict = solution_plan[0]
if plan_dict["primitive"] != "nova.restore_cluster":
log(" not nova.restore_cluster")
return [0, "null"]
log(" set_restore_list call")
api = api_from_models()
#get restore list from db
#restore_list=["data1", "data2","data3"]
#get db list
retdata = get_restoredb_list(node_id,api)
if 0 !=retdata[0]:
log(" get_restoredb_list err")
restore_list = retdata[1]
#sort new data is top
restore_list.sort(reverse=True)
if len(restore_list) != 0:
restore_key_list = make_restore_list_key(restore_list)
#make dictionary
#restore_list_dict={}
restore_list_dict = OrderedDict()
for x in range(len(restore_list)):
#restore_list_dict[restore_list[x]]=""
restore_list_dict[restore_key_list[x]]=restore_list[x]
solution_plan_dict["args"]["restore_name"]["list"] = restore_list_dict
log(" solution_plan_dict:%s" %(solution_plan_dict) )
return [1, [solution_plan_dict]]
def solve_for_node(node_id, constraints, api=None, plan=None):
"""
given a node id and a list of constraints, run a solver
to try and find a solution path.
it returns (is_solvable, requires_input, solution_plan)
"""
if api is None:
api = api_from_models()
if plan is not None:
task_solver = solver.Solver.from_plan(api, node_id, [], plan)
else:
task_solver = solver.Solver(api, node_id, constraints)
is_solvable, requires_input, solution_plan = task_solver.solve()
return (is_solvable, requires_input, solution_plan)
def execute_adventure(adventure_id):
data = flask.request.json
if not 'node' in data:
return generic.http_badrequest(msg='node not specified')
api = api_from_models()
try:
adventure = api._model_get_by_id('adventures', int(adventure_id))
except exceptions.IdNotFound:
message = 'Not Found: Adventure %s' % adventure_id
return generic.http_notfound(msg=message)
try:
return generic.http_solver_request(data['node'], [],
api=api, plan=adventure['dsl'])
except exceptions.IdNotFound:
#Can IdNotFound be raised for any other reason?
return generic.http_notfound(msg='Not Found: Node %s' % data['node'])
def downgrade(migrate_engine):
meta = MetaData(bind=migrate_engine)
api = api_from_models()
adventure_names = [x['name'] for x in adventures]
for name in adventure_names:
adventure_list = api._model_query('adventures', 'name="%s"' % name)
for adv in adventure_list:
api._model_delete_by_id('adventures', adv['id'])
node_list = ['"support"', '"unprovisioned"', '"workspace"']
for node in node_list:
tmp = api.nodes_query('name = %s' % node)
fact_list = api.facts_query('node_id = %s' % tmp[0]['id'])
for fact in fact_list:
api.fact_delete_by_id(fact['id'])
api.node_delete_by_id(tmp[0]['id'])
def f():
resp = None
builder = FilterBuilder(
FilterTokenizer(),
'%s: %s' % (what, request.json['filter']),
api=api_from_models())
# try:
result = builder.filter()
resp = jsonify({'status': 200,
'message': 'success',
what: result})
# except SyntaxError as e:
# resp = jsonify({'status': 400,
# 'message': 'Syntax error: %s' % e.msg})
# resp.status_code = 400
return resp
