def run_adventure(adventure_dsl=None, nodes=None): """ run an arbitrary adventure on a set of nodes, either by ID or as an ad-hoc adventure dsl. There are going to be issues with running ad-hoc adventures including logging and tracking problems. these should be straightened out. """ payload = {} adv_globals = {} payload['adventure_dsl'] = adventure_dsl payload['globals'] = adv_globals node_list = nodes api = api_from_models() # we will no longer expand node lists. At some point # we either need hints on adventures for whether they are # targetted at nodes or containers, or whether they # should be expanded. Or perhaps offer an API call # to expand node lists. Or something else altogether. # LOG.debug('node list before expansion: %s' % nodes) # node_list = expand_nodelist(nodes) # LOG.debug('node list after expansion: %s' % node_list) if len(node_list) == 0: raise ValueError('no nodes specified to run on') payload['nodes'] = node_list # find the node with the adventurator plugin query = "'adventurator' in attrs.opencenter_agent_output_modules" adventure_nodes = api.nodes_query(query) # find out how long this should run... max_time = 0 for step in adventure_dsl: max_time += step.get('timeout', 30) if len(adventure_nodes) > 0: adventure_node = adventure_nodes.pop(0)['id'] task = api.task_create({'action': 'adventurate', 'node_id': adventure_node, 'payload': payload, 'expires': int(time.time() + max_time)}) # FAIL: create a task update -- this should be done # via the model... task_semaphore = 'task-for-%s' % adventure_node notify(task_semaphore) else: raise ValueError('no adventurator') return task
def tasks_blocking_by_node_id(node_id): api = api_from_models() # README(shep): Using last_checkin attr for agent-health timestamp = int(time.time()) args = {'node_id': node_id, 'key': 'last_checkin', 'value': timestamp} try: r = api.attr_create(args) except exceptions.IdNotFound: message = 'Node %s not found.' % args['node_id'] return generic.http_notfound(msg=message) except exceptions.IdInvalid: return generic.http_badrequest() #DB does not hit updater, so we need to notify generic._update_transaction_id('nodes', id_list=[node_id]) generic._update_transaction_id('attrs', id_list=[r['id']]) while True: task = api.task_get_first_by_query("node_id=%d and state='pending'" % int(node_id)) if task is None: semaphore = 'task-for-%s' % node_id flask.current_app.logger.debug('waiting on %s' % semaphore) if not utility.wait(semaphore): flask.current_app.logger.error("ERROR ON WAIT") # utility.clear(semaphore) return generic.http_notfound(msg='no task found') else: flask.current_app.logger.error("SUCCESS ON WAIT") else: # utility.clear(semaphore) return generic.http_response(task=task)
def modify_fact(object_id): """ Express a fact modification as a list of constraints on the linked node.facts, and run the solved result """ data = flask.request.json api = api_from_models() model_object = api._model_get_by_id('facts', object_id) if not model_object: return generic.http_notfound() node_id = model_object['node_id'] # FIXME: TYPECASTING WHEN WE HAVE FACT TYPES constraints = ['facts.%s = "%s"' % (model_object['key'], data['value'])] return generic.http_solver_request( node_id, constraints, api=api, result={'fact': {'id': model_object['id'], 'node_id': node_id, 'key': model_object['key'], 'value': data['value']}})
def _clean_tasks(): """ clean up completed tasks over task_reaping_threshold seconds old. This is set in the [main] section of the config. The default is 300 seconds (5 min). This will not run more often than once per minute. """ current_time = time.time() if current_time < task_last_run + 60: return api = api_from_models() reaping_threshold = flask.current_app.config['task_reaping_threshold'] expiration_threshold = current_time - int(reaping_threshold) expired_tasks = api._model_query( object_type, '(state = "done" or state="cancelled") ' 'and completed < %d' % expiration_threshold) for task in expired_tasks: api._model_delete_by_id(object_type, task['id'])
def create(): old_fact = None # if we are creating with the same host_id and key, then we'll just update # fields = api._model_get_columns(object_type) api = api_from_models() data = flask.request.json model_object = None if 'node_id' in data and 'key' in data: old_fact = api._model_get_first_by_query( object_type, 'node_id=%d and key="%s"' % (int(data['node_id']), data['key'])) if old_fact: model_object = api._model_update_by_id( object_type, old_fact['id'], data) # send update notification generic._notify(model_object, object_type, old_fact['id']) else: try: model_object = api._model_create(object_type, data) except KeyError as e: # missing required field return generic.http_badrequest(msg=str(e)) generic._notify(model_object, object_type, model_object['id']) href = flask.request.base_url + str(model_object['id']) return generic.http_response(201, '%s Created' % singular_object_type.capitalize(), ref=href, **{singular_object_type: model_object})
def setUp(self): if opencenter.backends.primitive_by_name('test.set_test_fact') is None: opencenter.backends.load_specific_backend('tests.test', 'TestBackend') if opencenter.backends.primitive_by_name('test2.add_backend') is None: opencenter.backends.load_specific_backend('tests.test2', 'Test2Backend') self._clean_all() self.interfaces = {} self.adv = self._stub_node( 'adventurator', facts={'backends': ['node', 'agent']}) self.container = self._stub_node( 'container', facts={'backends': ['node', 'container']}) self.node = self._stub_node('node-1') chef_expr = '(facts.chef_server_uri != None) and ' \ '(facts.chef_server_pem != None)' # some of our current primitives require this self.interfaces['chef'] = self._model_create('filters', name='chef-server', filter_type='interface', expr=chef_expr) self.api = db_api.api_from_models() self.assertEquals(len(self._model_get_all('tasks')), 0)
def object_by_id(object_type, object_id): s_obj = singularize(object_type) api = api_from_models() if flask.request.method == 'PUT': # we just updated something, poke any waiters model_object = api._model_update_by_id(object_type, object_id, flask.request.json) _notify(model_object, object_type, object_id) return http_response(200, '%s Updated' % s_obj.capitalize(), **{s_obj: model_object}) elif flask.request.method == 'DELETE': try: if api._model_delete_by_id(object_type, object_id): return http_response(200, '%s deleted' % s_obj.capitalize()) _notify(None, object_type, object_id) except exceptions.IdNotFound: return http_notfound(msg='not found') elif flask.request.method == 'GET': if 'poll' in flask.request.args: # we're polling semaphore = '%s-id-%s' % (object_type, object_id) utility.wait(semaphore) try: model_object = api._model_get_by_id(object_type, object_id) except exceptions.IdNotFound: return http_notfound(msg='not found') return http_response(200, 'success', **{s_obj: model_object}) else: return http_notfound(msg='Unknown method %s' % flask.request.method)
def tasks_blocking_by_node_id(node_id): api = api_from_models() # README(shep): Using last_checkin attr for agent-health timestamp = int(time.time()) args = {'node_id': node_id, 'key': 'last_checkin', 'value': timestamp} r = api.attr_create(args) #DB does not hit updater, so we need to notify generic._update_transaction_id('nodes', id_list=[node_id]) generic._update_transaction_id('attrs', id_list=[r['id']]) task = api.task_get_first_by_query("node_id=%d and state='pending'" % int(node_id)) # README(shep): moving this out of a while loop, to let agent-health work semaphore = 'task-for-%s' % node_id flask.current_app.logger.debug('waiting on %s' % semaphore) utility.wait(semaphore) task = api.task_get_first_by_query("node_id=%d and state='pending'" % int(node_id)) if task: utility.clear(semaphore) result = flask.jsonify({'task': task}) else: result = generic.http_notfound(msg='no task found') return result
def run_plan(): # this comes in just like an optioned plan. We'll stuff any returned # args and call it a plan. <rimshot> # api = api_from_models() data = flask.request.json if not 'node' in data: return generic.http_badrequest(msg='no node specified') if not 'plan' in data: return generic.http_badrequest(msg='no plan specified') plan = data['plan'] # this is more than a bit awkward for step in plan: if 'args' in step: for arg in step['args']: if 'value' in step['args'][arg]: step['ns'][arg] = step['args'][arg]['value'] step.pop('args') # now our plan is a standard plan. Let's run it return generic.http_solver_request(data['node'], [], api=api, plan=plan)
def _expand_nodes(nodelist, filter_f=true_f, api=None, depth=0, detailed=False): if api is None: api = api_from_models() final_nodes = [] nodes = copy.deepcopy(nodelist) seen = {} for node in nodes: if isinstance(node, (int, long)): node = api.node_get_by_id(node) if not node['id'] in seen: seen[node['id']] = {"inspect": False, "level": 0} elif not seen[node['id']]['inspect']: #We've already inspected this node, move on. next #We're about to inspect the node. Don't do it if we see the node again seen[node['id']]['inspect'] = False if filter_f(node): if detailed: final_nodes.append(node) elif node is not None: final_nodes.append(node['id']) #We'll add new children provided we're not out of depth if is_container(node) and (depth == 0 or seen[node['id']]['level'] < depth): new_nodes = api.nodes_query('facts.parent_id = %s' % node['id']) for new_n in new_nodes: if not new_n['id'] in seen: seen[new_n['id']] = { "inspect": True, "level": seen[node['id']]['level'] + 1} nodes.append(new_n) return final_nodes
def f(filter_id): api = api_from_models() filter_obj = api.filter_get_by_id(filter_id) full_expr = filter_obj['full_expr'] builder = FilterBuilder(FilterTokenizer(), '%s: %s' % (what, full_expr)) return jsonify({what: builder.filter()})
def get_direct_children(node_id, api=None): """ given a node_id, return a list of all direct child nodes """ if api is None: api = api_from_models() return [x for x in _expand_nodes([node_id], api=api, depth=1, detailed=True) if x['id'] != node_id]
def fully_expand_nodelist(nodelist, api=None): """ given a list of nodes (including containers), generate a fully expanded list of all node_ids in node_list as well as their descendant nodes """ if api is None: api = api_from_models() return _expand_nodes(nodelist, api=api)
def expand_nodelist(nodelist, api=None): """ given a list of nodes (including containers), generate a fully expanded list of non-container-y nodes """ if api is None: api = api_from_models() return _expand_nodes(nodelist, api=api, filter_f=is_leaf)
def upgrade(migrate_engine): meta = MetaData(bind=migrate_engine) api = api_from_models() for adventure in adventures: new_adventure = {'name': adventure['name']} json_path = os.path.join( os.path.dirname(__file__), adventure['dsl']) criteria_path = os.path.join( os.path.dirname(__file__), adventure['criteria']) new_adventure['dsl'] = json.loads(open(json_path).read()) new_adventure['criteria'] = open(criteria_path).read() api.adventure_create(new_adventure) canned_filters = [{'name': 'unprovisioned nodes', 'filter_type': 'node', 'expr': 'backend=\'unprovisioned\''}, {'name': 'chef client nodes', 'filter_type': 'node', 'expr': 'backend=\'chef-client\''}, {'name': 'chef-server', 'filter_type': 'interface', 'expr': 'facts.chef_server_uri != None and ' 'facts.chef_server_pem != None'}] for new_filter in canned_filters: api._model_create('filters', new_filter) workspace = api.node_create({'name': 'workspace'}) api._model_create('attrs', {'node_id': workspace['id'], 'key': 'json_schema_version', 'value': 1}) unprov = api.node_create({'name': 'unprovisioned'}) api._model_create('facts', {'node_id': unprov['id'], 'key': 'parent_id', 'value': workspace['id']}) support = api.node_create({'name': 'support'}) api._model_create('facts', {'node_id': support['id'], 'key': 'parent_id', 'value': workspace['id']}) # Add default fact to the default nodes node_list = [(workspace, "Workspace"), (unprov, "Available Nodes"), (support, "Service Nodes")] for node, display in node_list: api.fact_create({'node_id': node['id'], 'key': 'backends', 'value': ["container", "node"]}) api.attr_create({'node_id': node['id'], 'key': 'display_name', 'value': display}) api.attr_create({'node_id': node['id'], 'key': 'locked', 'value': True})
def setUp(self): self.server = self._model_create('nodes', name='opencenter-server') self.agent = self._model_create('nodes', name='opencenter-client') for node_id in [self.server['id'], self.agent['id']]: self._make_facts(node_id, {'parent_id': 2, 'backends': ['agent', 'node']}) # grab the api so we can roll stuff forward self.api = db_api.api_from_models()
def get_objectlist_not_reserved_tenant(target_tenant): api = api_from_models() # get a nodes of mismatch attr tenant query = 'attrs.tenant!=null and attrs.tenant!="%s" and attrs.tenant!="any"' % target_tenant node_objects = api.nodes_query(query) # get a nodes of server and sdn box query = '"agent" in facts.backends and attrs.tenant!="any"' node_obj_agents = api.nodes_query(query) # get devide list that are assigned to the tenannt from the resouce managers ori=ool_rm_if.ool_rm_if() cookie_data = request.headers.get('Cookie') tokenId = '' cookie_split=cookie_data.split(';') for cookie in cookie_split: index = cookie.find(TOKENKEYWORD) if index != -1: tokenId = cookie[index + len(TOKENKEYWORD):] break if 0 == len(tokenId): logging.debug('facts_please.py:create() not find tokenID') return node_objects #logging.debug('TokenID=%s' % tokenId) #logging.debug('User-Agent=%s' % flask.request.headers.get('User-Agent')) #logging.debug('tenant=%s' % request.headers.get('tenant')) ori.set_auth(tokenId) data = ori.get_tenant('', target_tenant) #logging.debug('data = %s' % data) # return err if -1 == data[0]: logging.debug("ori.get_tenant err data=%s" %(data)) raise Invalid_Key # set result result = data[1] # Add to list server & SDN box that are not reserved by the tenant for obj in node_obj_agents: for node in result: if obj['name'] == node['device_name']: break else: #logging.debug('obj = %s' % obj) node_objects.append(obj) return node_objects
def solve_and_run(node_id, constraints, api=None, plan=None): if api is None: api = api_from_models() is_solvable, requires_input, solution_plan = solve_for_node( node_id, constraints, api=api, plan=plan) task = None if is_solvable: task = run_adventure(adventure_dsl=solution_plan, nodes=[node_id]) return task, is_solvable, requires_input, solution_plan
def tasks_by_node_id(node_id): api = api_from_models() # Display only tasks with state=pending task = api.task_get_first_by_query("node_id=%d and state='pending'" % int(node_id)) if not task: return generic.http_notfound() else: resp = generic.http_response(task=task) task['state'] = 'delivered' api._model_update_by_id('tasks', task['id'], task) return resp
def task_by_id(object_id): result = generic.object_by_id(object_type, object_id) if flask.request.method == "PUT": api = api_from_models() task = api.task_get_by_id(object_id) if "node_id" in task: flask.current_app.logger.debug("Task: %s" % task) task_semaphore = "task-for-%s" % task["node_id"] flask.current_app.logger.debug("notifying event %s" % task_semaphore) utility.notify(task_semaphore) return result
def task_by_id(object_id): result = generic.object_by_id(object_type, object_id) if flask.request.method == 'PUT': api = api_from_models() task = api.task_get_by_id(object_id) if 'node_id' in task: flask.current_app.logger.debug('Task: %s' % task) task_semaphore = 'task-for-%s' % task['node_id'] flask.current_app.logger.debug('notifying event %s' % task_semaphore) utility.notify(task_semaphore) return result
def list_index(): api = api_from_models() models = api._get_models() url = flask.request.url msg = {'url': flask.request.url, 'resources': {}} for model in models: msg['resources'][model] = {'url': '%s%s/' % (url, model)} resp = flask.jsonify(msg) resp.status_code = 200 return resp
def downgrade(migrate_engine): meta = MetaData(bind=migrate_engine) api = api_from_models() for adventure in adventures: db_entries = api._model_query('adventures', 'name="%s"' % adventure['name']) if len(db_entries) == 1: db_entry = db_entries[0] criteria_path = os.path.join( os.path.dirname(__file__), adventure['criteria']['001']) db_entry['criteria'] = open(criteria_path).read() api.adventure_update_by_id(db_entry['id'], db_entry)
def _notify(updated_object, object_type, object_id): semaphore = '%s-id-%s' % (object_type, object_id) utility.notify(semaphore) # TODO: Generalize the block of code with a TODO below here. # if updated_object is not None and object_type in related_notifications: # for field, entity in related_notifications[object_type].iteritems(): # if field in updated_object and updated_object[field] is not None: # semaphore = '%s-id-%s' % (entity, updated_object[field]) # utility.notify(semaphore) # TODO (wilk or rpedde): Use specific notifications for inheritance if object_type not in ('attrs', 'facts', 'nodes'): return try: node_id = updated_object['node_id'] node = None except KeyError: node_id = updated_object['id'] node = updated_object if object_type != "attrs": api = api_from_models() # We're just going to notify every child when containers are updated if node is None: try: node = api._model_get_by_id('nodes', node_id) except (exceptions.IdNotFound): return if 'container' in node['facts'].get('backends', []): children = utility.get_direct_children(node, api) for child in children: semaphore = 'nodes-id-%s' % child['id'] utility.notify(semaphore) # Update transaction for node and children id_list = utility.fully_expand_nodelist([node], api) # TODO(shep): this needs to be better abstracted # Need a codepath to update transaction for attr modifications else: # TODO(shep): this needs to be better abstracted id_list = [node_id] _update_transaction_id('nodes', id_list)
def create(): old_fact = None # if we are creating with the same host_id and key, then we'll just update # fields = api._model_get_columns(object_type) api = api_from_models() data = flask.request.json try: node_id = data['node_id'] key = data['key'] except TypeError: return generic.http_badrequest('node_id and key are required.') except KeyError: pass else: query = 'node_id=%d and key="%s"' % (int(node_id), key) old_fact = api._model_get_first_by_query(object_type, query) if old_fact: return modify_fact(old_fact['id']) # here, if the fact is a fact on a container, # we need to solve for fact application on all # child nodes. <eek> # # FIXME(rp): so we'll punt for now, and just refuse fact # creates on containers. children = api._model_query('nodes', 'facts.parent_id = %s' % data['node_id']) if len(children) > 0: return generic.http_response(403, msg='cannot set fact on containers', friendly='oopsie') constraints = ['facts.%s = "%s"' % (data['key'], data['value'])] return generic.http_solver_request( data['node_id'], constraints, api=api, result={'fact': {'id': -1, 'node_id': data['node_id'], 'key': data['key'], 'value': data['value']}})
def set_restore_list(plan, solution_plan, node_id): plan_dict = plan[0] solution_plan_dict = solution_plan[0] if plan_dict["primitive"] != "nova.restore_cluster": log(" not nova.restore_cluster") return [0, "null"] log(" set_restore_list call") api = api_from_models() #get restore list from db #restore_list=["data1", "data2","data3"] #get db list retdata = get_restoredb_list(node_id,api) if 0 !=retdata[0]: log(" get_restoredb_list err") restore_list = retdata[1] #sort new data is top restore_list.sort(reverse=True) if len(restore_list) != 0: restore_key_list = make_restore_list_key(restore_list) #make dictionary #restore_list_dict={} restore_list_dict = OrderedDict() for x in range(len(restore_list)): #restore_list_dict[restore_list[x]]="" restore_list_dict[restore_key_list[x]]=restore_list[x] solution_plan_dict["args"]["restore_name"]["list"] = restore_list_dict log(" solution_plan_dict:%s" %(solution_plan_dict) ) return [1, [solution_plan_dict]]
def solve_for_node(node_id, constraints, api=None, plan=None): """ given a node id and a list of constraints, run a solver to try and find a solution path. it returns (is_solvable, requires_input, solution_plan) """ if api is None: api = api_from_models() if plan is not None: task_solver = solver.Solver.from_plan(api, node_id, [], plan) else: task_solver = solver.Solver(api, node_id, constraints) is_solvable, requires_input, solution_plan = task_solver.solve() return (is_solvable, requires_input, solution_plan)
def execute_adventure(adventure_id): data = flask.request.json if not 'node' in data: return generic.http_badrequest(msg='node not specified') api = api_from_models() try: adventure = api._model_get_by_id('adventures', int(adventure_id)) except exceptions.IdNotFound: message = 'Not Found: Adventure %s' % adventure_id return generic.http_notfound(msg=message) try: return generic.http_solver_request(data['node'], [], api=api, plan=adventure['dsl']) except exceptions.IdNotFound: #Can IdNotFound be raised for any other reason? return generic.http_notfound(msg='Not Found: Node %s' % data['node'])
def downgrade(migrate_engine): meta = MetaData(bind=migrate_engine) api = api_from_models() adventure_names = [x['name'] for x in adventures] for name in adventure_names: adventure_list = api._model_query('adventures', 'name="%s"' % name) for adv in adventure_list: api._model_delete_by_id('adventures', adv['id']) node_list = ['"support"', '"unprovisioned"', '"workspace"'] for node in node_list: tmp = api.nodes_query('name = %s' % node) fact_list = api.facts_query('node_id = %s' % tmp[0]['id']) for fact in fact_list: api.fact_delete_by_id(fact['id']) api.node_delete_by_id(tmp[0]['id'])
def f(): resp = None builder = FilterBuilder( FilterTokenizer(), '%s: %s' % (what, request.json['filter']), api=api_from_models()) # try: result = builder.filter() resp = jsonify({'status': 200, 'message': 'success', what: result}) # except SyntaxError as e: # resp = jsonify({'status': 400, # 'message': 'Syntax error: %s' % e.msg}) # resp.status_code = 400 return resp