Example #1
0
def _is_node_busy_and_reserve_it(node_id, api, controller_requestor='gcc'):
    settings = get_current_registry().settings
    seconds_block_is_busy = int(settings.get('chef.seconds_block_is_busy'))
    time_to_exp = datetime.timedelta(seconds=seconds_block_is_busy)
    node = ChefNode(node_id, api)
    current_use_node = node.attributes.get(USE_NODE, {})
    current_use_node_control = current_use_node.get('control', None)
    current_use_node_exp_date = current_use_node.get('exp_date', None)
    if current_use_node_exp_date:
        current_use_node_exp_date = json.loads(
            current_use_node_exp_date, object_hook=json_util.object_hook)
        current_use_node_exp_date = current_use_node_exp_date.astimezone(
            pytz.utc).replace(tzinfo=None)
        now = datetime.datetime.now()
        if now - current_use_node_exp_date > time_to_exp:
            current_use_node_control = None
    if current_use_node_control == controller_requestor:
        return (node, False)
    elif current_use_node_control is None:
        exp_date = datetime.datetime.utcnow() + time_to_exp
        node.attributes.set_dotted(
            USE_NODE, {
                'control': controller_requestor,
                'exp_date': json.dumps(exp_date, default=json_util.default)
            })
        node.save()
        node2 = ChefNode(node.name, api)  # second check
        current_use_node2 = node2.attributes.get(USE_NODE, {})
        current_use_control2 = current_use_node2.get('control', None)
        if current_use_control2 == controller_requestor:
            return (node2, False)
    return (node, True)
Example #2
0
def build_computes(computes):
    # Run computes
    print "Making the compute nodes..."
    for compute in computes:
        compute_node = Node(compute)
        compute_node['in_use'] = "compute"
        compute_node.run_list = ["role[qa-single-compute]"]
        compute_node.save()

        print "Updating server...this may take some time"
        update_node(compute_node)

        if compute_node['platform_family'] == 'rhel':
            print "Platform is RHEL family, disabling iptables"
            disable_iptables(compute_node)

        # Run chef client twice
        print "Running chef-client on compute node: %s, this may take some time..." % compute
        run1 = run_chef_client(compute_node)
        if run1['success']:
            print "First chef-client run successful...starting second run..."
            run2 = run_chef_client(compute_node)
            if run2['success']:
                print "Second chef-client run successful..."
            else:
                print "Error running chef-client for compute %s" % compute
                print run2
                sys.exit(1)
        else:
            print "Error running chef-client for compute %s" % compute
            print run1
            sys.exit(1)
Example #3
0
def handle(event, _context):
    """Lambda Handler"""
    log_event(event)

    with ChefAPI(CHEF_SERVER_URL, get_pem(), USERNAME):
        instance_id = get_instance_id(event)
        try:
            search = Search('node', 'ec2_instance_id:' + instance_id)
        except ChefServerNotFoundError as err:
            LOGGER.error(err)
            return False

        if len(search) != 0:
            for instance in search:
                node = Node(instance.object.name)
                client = Client(instance.object.name)
                try:
                    node.delete()
                    LOGGER.info('===Node Delete: SUCCESS===')
                    client.delete()
                    LOGGER.info('===Client Delete: SUCCESS===')
                    return True
                except ChefServerNotFoundError as err:
                    LOGGER.error(err)
                    return False
        else:
            LOGGER.info('=Instance does not appear to be Chef Server managed.=')
            return True
    def post(self):
        node_id = self.request.POST.get('node_id')
        if node_id is None:
            return {'ok': False, 'message': 'Missing node ID'}

        settings = get_current_registry().settings
        api = get_chef_api(settings, self.request.user)

        # create chef client
        chef_client = ChefClient(node_id, api)
        if chef_client.exists:
            return {'ok': False, 'message': 'This client already exists'}

        chef_client = ChefClient.create(node_id, api)
        
        # Prepare the API for this client
        chef_url = settings.get('chef.url')
        chef_version = settings.get('chef.version')
        chef_ssl_verify = settings.get('chef.ssl.verify')
        if chef_ssl_verify == 'False' or chef_ssl_verify == 'True':
            chef_ssl_verify = bool(chef_ssl_verify)
        api = ChefAPI(chef_url, str(chef_client.private_key), node_id, chef_version, ssl_verify = False)

 
        # create chef node
        chef_node = ChefNode(node_id, api)
        if chef_node.exists:
            return {'ok': False, 'message': 'This node already exists'}
        chef_node.save()

        return {'ok': True, 'message': 'Node and client have been added',
                'client_private_key': chef_client.private_key}
Example #5
0
 def chef_instance(self, deployment, name, flavor="2GBP"):
     """
     Builds an instance with desired specs and inits it with chef
     :param client: compute client object
     :type client: novaclient.client.Client
     :param deployment: deployement to add to
     :type deployment: ChefDeployment
     :param name: name for instance
     :type name: string
     :param flavor: desired flavor for node
     :type flavor: string
     :rtype: ChefNode
     """
     image = deployment.os_name
     server, password = self.build_instance(name=name, image=image,
                                            flavor=flavor)
     run_list = ",".join(util.config[str(self)]['run_list'])
     run_list_arg = ""
     if run_list:
         run_list_arg = "-r {0}".format(run_list)
     command = 'knife bootstrap {0} -u root -P {1} -N {2} {3}'.format(
         server.accessIPv4, password, name, run_list_arg)
     run_cmd(command)
     node = Node(name, api=deployment.environment.local_api)
     node.chef_environment = deployment.environment.name
     node['in_use'] = "provisioning"
     node['ipaddress'] = server.accessIPv4
     node['password'] = password
     node['uuid'] = server.id
     node['current_user'] = "******"
     node.save()
     return node
Example #6
0
def _is_node_busy_and_reserve_it(node_id, api, controller_requestor='gcc'):
    settings = get_current_registry().settings
    seconds_block_is_busy = int(settings.get('chef.seconds_block_is_busy'))
    time_to_exp = datetime.timedelta(seconds=seconds_block_is_busy)
    node = ChefNode(node_id, api)
    current_use_node = node.attributes.get(USE_NODE, {})
    current_use_node_control = current_use_node.get('control', None)
    current_use_node_exp_date = current_use_node.get('exp_date', None)
    if current_use_node_exp_date:
        current_use_node_exp_date = json.loads(current_use_node_exp_date, object_hook=json_util.object_hook)
        current_use_node_exp_date = current_use_node_exp_date.astimezone(pytz.utc).replace(tzinfo=None)
        now = datetime.datetime.now()
        if now - current_use_node_exp_date > time_to_exp:
            current_use_node_control = None
    if current_use_node_control == controller_requestor:
        return (node, False)
    elif current_use_node_control is None:
        exp_date = datetime.datetime.utcnow() + time_to_exp
        node.attributes.set_dotted(USE_NODE, {'control': controller_requestor,
                                              'exp_date': json.dumps(exp_date, default=json_util.default)})
        node.save()
        node2 = ChefNode(node.name, api)  # second check
        current_use_node2 = node2.attributes.get(USE_NODE, {})
        current_use_control2 = current_use_node2.get('control', None)
        if current_use_control2 == controller_requestor:
            return (node2, False)
    return (node, True)
Example #7
0
def create_nodes(cluster, facet):
  """Initialize Chef nodes"""

  instances = create_instances(cluster, facet)

  for nodename, ipaddress in instances:
    node = Node(nodename)
    if node.exists:
      node_ipaddress = node.get('ipaddress')
      if ipaddress is None and node_ipaddress:
        ipaddress = node_ipaddress
      elif node_ipaddress and node_ipaddress != ipaddress:
        raise Exception('The remote IP address is different: %s' % node_ipaddress)

    if ipaddress is None:
      raise Exception('Can not determine the IP address for %s' % nodename)

    node['ipaddress'] = ipaddress

    # update environment and run_list
    node.chef_environment = cluster.environment

    run_list = list(cluster.run_list)
    run_list.extend(facet.run_list)

    # tagging the cluster
    run_list.append(u'role[%s_cluster]' % cluster.name)
    run_list.append(u'role[%s_%s]'% (cluster.name, facet.name))

    for role in run_list:
      if role not in node.run_list:
        node.run_list.append(role)

    facet.nodes[ipaddress] = node
    def post(self):
        node_id = self.request.POST.get('node_id')
        if node_id is None:
            return {'ok': False, 'message': 'Missing node ID'}

        settings = get_current_registry().settings
        api = get_chef_api(settings, self.request.user)

        # create chef client
        chef_client = ChefClient(node_id, api)
        if chef_client.exists:
            return {'ok': False, 'message': 'This client already exists'}

        chef_client = ChefClient.create(node_id, api)
        
        # Prepare the API for this client
        chef_url = settings.get('chef.url')
        chef_version = settings.get('chef.version')
        chef_ssl_verify = settings.get('chef.ssl.verify')
        if chef_ssl_verify == 'False' or chef_ssl_verify == 'True':
            chef_ssl_verify = bool(chef_ssl_verify)
        api = ChefAPI(chef_url, chef_client.private_key.encode(), node_id, chef_version, ssl_verify = False)

 
        # create chef node
        chef_node = ChefNode(node_id, api)
        if chef_node.exists:
            return {'ok': False, 'message': 'This node already exists'}
        chef_node.save()

        return {'ok': True, 'message': 'Node and client have been added',
                'client_private_key': chef_client.private_key}
Example #9
0
def build_dir_server(dir_server):
    # We dont support 389 yet, so exit if it is not ldap
    if results.dir_version != 'openldap':
        print "%s as a directory service is not yet supported...exiting" % results.dir_version
        sys.exit(1)

    # Build directory service node
    dir_node = Node(dir_server)
    ip = dir_node['ipaddress']
    root_pass = razor.get_active_model_pass(dir_node['razor_metadata'].to_dict()['razor_active_model_uuid'])['password']
    dir_node['in_use'] = 'directory-server'
    dir_node.run_list = ["role[qa-%s-%s]" % (results.dir_version, results.os)]
    dir_node.save()

    print "Updating server...this may take some time"
    update_node(dir_node)

    # if redhat platform, disable iptables
    if dir_node['platform_family'] == 'rhel':
        print "Platform is RHEL family, disabling iptables"
        disable_iptables(dir_node)

    # Run chef-client twice
    print "Running chef-client for directory service node...this may take some time..."
    run1 = run_chef_client(dir_node)
    if run1['success']:
        print "First chef-client run successful...starting second run..."
        run2 = run_chef_client(dir_node)
        if run2['success']:
            print "Second chef-client run successful..."
        else:
            print "Error running chef-client for directory node %s" % dir_node
            print run2
            sys.exit(1)
    else:
        print "Error running chef-client for directory node %s" % dir_node
        print run1
        sys.exit(1)

    # Directory service is set up, need to import config
    if run1['success'] and run2['success']:
        if results.dir_version == 'openldap':
            scp_run = run_remote_scp_cmd(ip, 'root', root_pass, '/var/lib/jenkins/source_files/ldif/*.ldif')
            if scp_run['success']:
                ssh_run = run_remote_ssh_cmd(ip, 'root', root_pass, 'ldapadd -x -D \"cn=admin,dc=dev,dc=rcbops,dc=me\" -f base.ldif -w@privatecloud')
        elif results.dir_version == '389':
            # Once we support 389, code here to import needed config files
            print "389 is not yet supported..."
            sys.exit(1)
        else:
            print "%s is not supported...exiting" % results.dir_version
            sys.exit(1)

    if scp_run['success'] and ssh_run['success']:
        print "Directory Service: %s successfully set up..." % results.dir_version
    else:
        print "Failed to set-up Directory Service: %s..." % results.dir_version
        sys.exit(1)
    def test_create(self):
        name = self.random()
        node = Node.create(name, run_list=['recipe[foo]'])
        self.register(node)
        self.assertEqual(node.run_list, ['recipe[foo]'])

        node2 = Node(name)
        self.assertTrue(node2.exists)
        self.assertEqual(node2.run_list, ['recipe[foo]'])
Example #11
0
 def computer_deleted(self, user, obj, computers=None):
     node_chef_id = obj.get('node_chef_id', None)
     if node_chef_id:
         api = get_chef_api(self.app.conf, user)
         node = Node(node_chef_id, api)
         node.delete()
         client = Client(node_chef_id, api=api)
         client.delete()
     self.log_action('deleted', 'Computer', obj)
Example #12
0
 def add_run_list_item(self, items):
     """
     Adds list of items to run_list
     """
     util.logger.debug("run_list:{0} add:{1}".format(self.run_list, items))
     self.run_list.extend(items)
     cnode = ChefNode(self.name, api=self.environment.local_api)
     cnode.run_list = self.run_list
     self.save(cnode)
Example #13
0
 def computer_deleted(self, user, obj, computers=None):
     node_chef_id = obj.get('node_chef_id', None)
     if node_chef_id:
         api = get_chef_api(self.app.conf, user)
         node = Node(node_chef_id, api)
         node.delete()
         client = Client(node_chef_id, api=api)
         client.delete()
     self.log_action('deleted', 'Computer', obj)
Example #14
0
    def build(self):
        """ Builds the node
        """

        # clear run_list
        self.run_list = []
        node = ChefNode(self.name, self.environment.local_api)
        node.run_list = []
        node.save()
        super(Chef, self).build()
Example #15
0
 def remove_run_list_item(self, item):
     """
     Adds list of items to run_list
     """
     util.logger.debug("run_list:{0} remove:{1}".format(self.run_list,
                                                        item))
     self.run_list.pop(self.run_list.index(item))
     cnode = ChefNode(self.name, api=self.environment.local_api)
     cnode.run_list = self.run_list
     self.save(cnode)
Example #16
0
def clear_pool(chef_nodes, environment):
    for n in chef_nodes:
        name = n['name']
        node = Node(name)
        if node.chef_environment == environment:
            if "recipe[network-interfaces]" not in node.run_list:
                erase_node(name)
            else:
                node.chef_environment = "_default"
                node.save()
Example #17
0
def hosts(env=None, name=None):
    api = autoconfigure()
    if name:
        nodes = (node for node in Node.list() if name in Node(node).name)
    else:
        nodes = (node for node in Node.list() if Node(node).chef_environment == env)

    file = open("hosts", "w")
    file.write("[hosts]\n")

    map(lambda n: file.write("{0}\n".format(Node(n)['ipaddress'])), nodes)
def handle(event, _context):
    """Lambda Handler"""
    log_event(event)
    node_name = None
    node_ip = None

    # Remove from one of the chef servers
    for URL in CHEF_SERVER_URLS:
        with ChefAPI(URL, CHEF_PEM, CHEF_USERNAME):
            instance_id = get_instance_id(event)
            try:
                search = Search('node', 'ec2_instance_id:' + instance_id)
            except ChefServerNotFoundError as err:
                LOGGER.error(err)
                return False

            if len(search) != 0:
                for instance in search:
                    node_name = instance.object.name
                    node = Node(node_name)
                    node_ip = node['ipaddress']
                    client = Client(node_name)

                    try:
                        node.delete()
                        client.delete()
                        LOGGER.info(
                            '=====SUCCESSFULLY REMOVED INSTANCE FROM CHEF SERVER===== {}'
                            .format(URL))
                        break
                    except ChefServerNotFoundError as err:
                        LOGGER.error(err)
                        return False
            else:
                LOGGER.info(
                    '===Instance does not appear to be Chef Server managed.=== {}'
                    .format(URL))

    # Remove from Spacewalk
    spacewalk_cleanup(node_ip)

    # Remove from DNS
    dns_cleanup(node_name)

    # Remove from AD
    active_directory_cleanup(node_name)

    # Remove fom Solarwinds
    solarwinds_cleanup(node_ip, node_name)

    # Remove from Chef Automate
    chef_automate_cleanup(node_name)
Example #19
0
 def destroy_node(self, node):
     """
     Destroys chef node from openstack
     :param node: node to destroy
     :type node: ChefNode
     """
     cnode = Node(node.name, node.environment.local_api)
     if cnode.exists:
         self.compute_client.servers.get(node['uuid']).delete()
         cnode.delete()
     client = Client(node.name, node.environment.local_api)
     if client.exists:
         client.delete()
Example #20
0
def opencenter_endpoint(chef, name='test', os='ubuntu'):
    validate_environment(chef, name=name, os=os)
    env = env_format % (name, os)
    query = "in_use:\"server\" AND chef_environment:%s" % env
    server = next(Node(node['name']) for node in Search('node').query(query))
    ep_url = "https://%s:8443" % server['ipaddress']
    return OpenCenterEndpoint(ep_url, user="******", password="******")
Example #21
0
 def _clean_node(self, hostid, config, target_system, **kwargs):
     """clean node"""
     from chef import Node
     try:
         node = Node(self._get_node_name(config['hostname'],
                                         config['clusterid'],
                                         target_system),
                     api=self.api_)
         node.delete()
         logging.debug(
             'node is removed for host %s '
             'config %s target_system %s', hostid, config, target_system)
     except Exception as error:
         logging.debug(
             'no node to delete for host %s '
             'config %s target_system %s', hostid, config, target_system)
Example #22
0
class MockSearchTestCase(ChefTestCase):
    @mockSearch({('node', '*:*'): [Node('fake_1', skip_load=True).to_dict()]})
    def test_single_node(self, MockSearch):
        import chef.search
        s = chef.search.Search('node')
        self.assertEqual(len(s), 1)
        self.assertIn('fake_1', s)
 def test_create_crosslink(self):
     node = Node.create(self.random())
     self.register(node)
     node.normal['foo'] = 'bar'
     self.assertEqual(node['foo'], 'bar')
     node.attributes['foo'] = 'baz'
     self.assertEqual(node.normal['foo'], 'baz')
Example #24
0
 def _clean_node(self, hostid, config, target_system, **kwargs):
     """clean node"""
     from chef import Node
     try:
         node = Node(
             self._get_node_name(
                 config['hostname'], config['clusterid'], target_system),
             api=self.api_
         ) 
         node.delete()
         logging.debug('node is removed for host %s '
                       'config %s target_system %s',
                       hostid, config, target_system)
     except Exception as error:
         logging.debug('no node to delete for host %s '
                       'config %s target_system %s',
                       hostid, config, target_system)
Example #25
0
 def destroy_node(self, node):
     """
     Destroys a node provisioned by razor
     :param node: Node to destroy
     :type node: ChefNode
     """
     cnode = Node(node.name, node.environment.local_api)
     in_use = node['in_use']
     if in_use == "provisioning" or in_use == 0:
         # Return to pool if the node is clean
         cnode['in_use'] = 0
         cnode['archive'] = {}
         cnode.chef_environment = "_default"
         cnode.save()
     else:
         # Remove active model if the node is dirty
         active_model = cnode['razor_metadata']['razor_active_model_uuid']
         try:
             if node.feature_in('controller'):
                 # rabbit can cause the node to not actually reboot
                 kill = ("for i in `ps -U rabbitmq | tail -n +2 | "
                         "awk '{print $1}' `; do kill -9 $i; done")
                 node.run_cmd(kill)
             node.run_cmd("shutdown -r now")
             self.api.remove_active_model(active_model)
             Client(node.name).delete()
             cnode.delete()
             sleep(15)
         except:
             util.logger.error("Node unreachable. "
                               "Manual restart required:{0}".format(
                                   str(node)))
    def delete(self):
        node_id = self.request.GET.get('node_id')
        if node_id is None:
            return {'ok': False, 'message': 'Missing node ID'}
        
        settings = get_current_registry().settings
        api = get_chef_api(settings, self.request.user)

        chef_node = ChefNode(node_id, api)
        if not chef_node.exists:
            return {'ok': False, 'message': 'This node does not exists'}
        chef_node.delete()

        chef_client = ChefClient(node_id, api)
        if not chef_client.exists:
            return {'ok': False, 'message': 'This client does not exists'}
        chef_client.delete()

        return {'ok': True, 'message': 'Node and client have been deleted'}
    def delete(self):
        node_id = self.request.GET.get('node_id')
        if node_id is None:
            return {'ok': False, 'message': 'Missing node ID'}
        
        settings = get_current_registry().settings
        api = get_chef_api(settings, self.request.user)

        chef_node = ChefNode(node_id, api)
        if not chef_node.exists:
            return {'ok': False, 'message': 'This node does not exists'}
        chef_node.delete()

        chef_client = ChefClient(node_id, api)
        if not chef_client.exists:
            return {'ok': False, 'message': 'This client does not exists'}
        chef_client.delete()

        return {'ok': True, 'message': 'Node and client have been deleted'}
Example #28
0
 def destroy_node(self, node):
     """
     Destroys a node provisioned by razor
     :param node: Node to destroy
     :type node: ChefNode
     """
     cnode = Node(node.name, node.environment.local_api)
     in_use = node['in_use']
     if in_use == "provisioning" or in_use == 0:
         # Return to pool if the node is clean
         cnode['in_use'] = 0
         cnode['archive'] = {}
         cnode.chef_environment = "_default"
         cnode.save()
     else:
         # Remove active model if the node is dirty
         active_model = cnode['razor_metadata']['razor_active_model_uuid']
         try:
             if node.feature_in('controller'):
                 # rabbit can cause the node to not actually reboot
                 kill = ("for i in `ps -U rabbitmq | tail -n +2 | "
                         "awk '{print $1}' `; do kill -9 $i; done")
                 node.run_cmd(kill)
             node.run_cmd("shutdown -r now")
             self.api.remove_active_model(active_model)
             Client(node.name).delete()
             cnode.delete()
             sleep(15)
         except:
             util.logger.error("Node unreachable. "
                               "Manual restart required:{0}".
                               format(str(node)))
Example #29
0
def _is_node_busy_and_reserve_it(node_id, api, controller_requestor='gcc'):
    '''
    Check if the node is busy, else try to get it and write in control and expiration date in the field USE_NODE.
    '''
    settings = get_current_registry().settings
    seconds_block_is_busy = int(settings.get('chef.seconds_block_is_busy'))
    time_to_exp = datetime.timedelta(seconds=seconds_block_is_busy)

    time_get = time.time()
    node = ChefNode(node_id, api)
    time_get = time.time() - time_get

    current_use_node = node.attributes.get(USE_NODE, {})
    current_use_node_control = current_use_node.get('control', None)
    current_use_node_exp_date = current_use_node.get('exp_date', None)
    if current_use_node_exp_date:
        current_use_node_exp_date = json.loads(current_use_node_exp_date, object_hook=json_util.object_hook)
        current_use_node_exp_date = current_use_node_exp_date.astimezone(pytz.utc).replace(tzinfo=None)
        now = datetime.datetime.now()
        if now - current_use_node_exp_date > time_to_exp:
            current_use_node_control = None
    if current_use_node_control == controller_requestor:
        return (node, False)
    elif current_use_node_control is None:
        exp_date = datetime.datetime.utcnow() + time_to_exp
        node.attributes.set_dotted(USE_NODE, {'control': controller_requestor,
                                              'exp_date': json.dumps(exp_date, default=json_util.default)})
        node.save()

        smart_lock_sleep_parameter = settings.get('chef.smart_lock_sleep_factor', 3)
        seconds_sleep_is_busy = time_get * int(smart_lock_sleep_parameter)
        time.sleep(seconds_sleep_is_busy)

        node2 = ChefNode(node.name, api)  # second check
        current_use_node2 = node2.attributes.get(USE_NODE, {})
        current_use_control2 = current_use_node2.get('control', None)
        if current_use_control2 == controller_requestor:
            return (node2, False)
    return (node, True)
Example #30
0
def openstack_endpoints(opencenter_endpoint):
    ep = opencenter_endpoint
    infrastructure_nodes = ep.nodes.filter('name = "Infrastructure"')
    for node_id in infrastructure_nodes.keys():
        ha = infrastructure_nodes[node_id].facts["ha_infra"]
        endpoint = None
        if ha:
            endpoint = infrastructure_nodes[node_id].facts["nova_api_vip"]
        else:
            name = next(node.name for node in ep.nodes
                        if "nova-controller" in node.facts["backends"])
            endpoint = Node(name)['ipaddress']
        return endpoint
Example #31
0
 def command(self):
     api = _get_chef_api(self.settings.get('chef.url'),
                         toChefUsername(self.options.chef_username),
                         self.options.chef_pem, self.settings.get('chef.ssl.verify'), self.settings.get('chef.version'))
     ou_name = 'ou_0'
     ou = self.create_root_ou(ou_name)
     for node_id in ChefNode.list():
         try:
             node_mongo_id = register_or_updated_node(api, node_id, ou, self.db.nodes)
         except DuplicateKeyError:
             node_mongo_id = update_node(api, node_id, ou, self.db.nodes)
         if not node_mongo_id:
             print "%s does not exists" % node_id
Example #32
0
def build_controller(controller, ha=False, ha_num=0):
    controller_node = Node(controller)

    # Check for ha
    if ha:
        print "Making %s the ha-controller%s node" % (controller, ha_num)
        controller_node['in_use'] = "ha-controller%s" % ha_num
        controller_node.run_list = ["role[qa-ha-controller%s]" % ha_num]
    else:
        print "Making %s the controller node" % controller
        controller_node['in_use'] = "controller"
        controller_node.run_list = ["role[qa-single-controller]"]
    # save node
    controller_node.save()

    print "Updating server...this may take some time"
    update_node(controller_node)

    if controller_node['platform_family'] == 'rhel':
        print "Platform is RHEL family, disabling iptables"
        disable_iptables(controller_node)

    # Run chef-client twice
    print "Running chef-client for controller node...this may take some time..."
    run1 = run_chef_client(controller_node)
    if run1['success']:
        print "First chef-client run successful...starting second run..."
        run2 = run_chef_client(controller_node)
        if run2['success']:
            print "Second chef-client run successful..."
        else:
            print "Error running chef-client for controller %s" % controller
            print run2
            sys.exit(1)
    else:
        print "Error running chef-client for controller %s" % controller
        print run1
        sys.exit(1)
Example #33
0
    def put(self):
        node_id = self.request.POST.get('node_id')
        username = self.request.POST.get('gcc_username')
        if not node_id:
            return {'ok': False,
                    'message': 'Please set a node id (node_id)'}
        if not username:
            return {'ok': False,
                    'message': 'Please set a admin username (gcc_username)'}
        self.request.user = self.request.db.adminusers.find_one({'username': username})
        if not self.request.user:
            return {'ok': False,
                    'message': 'The admin user %s does not exists' % username}
        settings = get_current_registry().settings
        api = get_chef_api(settings, self.request.user)
        node = Node(node_id, api)
        job_status = node.attributes.get('job_status')
        reserve_node = False
        if job_status:
            node = reserve_node_or_raise(node_id, api, 'gcc-chef-status-%s' % random.random(), attempts=3)
            reserve_node = True
            chef_client_error = False

            for job_id, job_status in job_status.to_dict().items():
                job = self.collection.find_one({'_id': ObjectId(job_id)})
                if not job:
                    continue
                if job_status['status'] == 0:
                    self.collection.update({'_id': job['_id']},
                                           {'$set': {'status': 'finished',
                                                     'last_update': datetime.datetime.utcnow()}})
                else:
                    chef_client_error = True
                    self.collection.update({'_id': job['_id']},
                                           {'$set': {'status': 'errors',
                                                     'message': job_status.get('message', 'Error'),
                                                     'last_update': datetime.datetime.utcnow()}})
            self.request.db.nodes.update({'node_chef_id': node_id}, {'$set': {'error_last_chef_client': chef_client_error}})
            invalidate_jobs(self.request)
            node.attributes.set_dotted('job_status', {})

        users_old = self.get_attr(node, USERS_OLD)
        users = self.get_attr(node, USERS_OHAI)
        if not users_old or users_old != users:
            if not reserve_node:
                node = reserve_node_or_raise(node_id, api, 'gcc-chef-status-%s' % random.random(), attempts=3)
            return self.check_users(node)
        if job_status:
            save_node_and_free(node)
        return {'ok': True}
Example #34
0
 def command(self):
     api = _get_chef_api(self.settings.get('chef.url'),
                         toChefUsername(self.options.chef_username),
                         self.options.chef_pem)
     ou_name = 'ou_0'
     ou = self.create_root_ou(ou_name)
     for node_id in ChefNode.list():
         try:
             node_mongo_id = register_or_updated_node(
                 api, node_id, ou, self.db.nodes)
         except DuplicateKeyError:
             node_mongo_id = update_node(api, node_id, ou, self.db.nodes)
         if not node_mongo_id:
             print "%s does not exists" % node_id
Example #35
0
    def fqdn_for_node(self, node):
        self.node = node
        api = autoconfigure()
        n = Node(node)
        if n:
            try:
                self.fqdn = n.attributes['ec2']['public_hostname']
            except KeyError:
                if 'fqdn' in n:
                    self.fqdn = n['fqdn']
                else:
                    return None
        else:
            return None

        return self.fqdn
 def get_node_information(self, node_name):
     """
     Get node attributes(metadata) using Node.attributes of PyChef
     Store the values of the attributes selected by the user for each node
     """
     chefUniqueId = self.organization + "_" + node_name
     node_details = Node(node_name)
     node_information = {}
     node_information['chefUniqueId'] = chefUniqueId
     node_information['chef_environment'] = node_details.chef_environment
     for attribute in self.config:
         attribute_value = self.get_attribute_value(attribute, node_details)
         if attribute_value:
             attribute = self.adjust_attribute_name(attribute)
             node_information[attribute] = attribute_value
     self.nodes_metadata.append(node_information)
Example #37
0
 def __setitem__(self, item, value):
     """
     Node can set chef attributes
     """
     lnode = CNode(self.name, api=self.environment.local_api)
     lnode[item] = value
     lnode.save()
     if self.environment.remote_api:
         rnode = CNode(self.name, api=self.environment.remote_api)
         rnode[item] = value
         rnode.save()
Example #38
0
 def __setitem__(self, item, value):
     """
     Node can set chef attributes
     """
     lnode = CNode(self.name, api=self.environment.local_api)
     lnode[item] = value
     lnode.save()
     if self.environment.remote_api:
         rnode = CNode(self.name, api=self.environment.remote_api)
         rnode[item] = value
         rnode.save()
Example #39
0
 def command(self):
     api = _get_chef_api(self.settings.get('chef.url'),
                         toChefUsername(self.options.chef_username),
                         self.options.chef_pem, self.settings.get('chef.version'))
                         
     print('INFO: Update IPv4 address START!') 
     db = self.pyramid.db
     computers = db.nodes.find({'type': 'computer'})
     for comp in computers:
         node_id = comp.get('node_chef_id', None)
         node = Node(node_id, api)
         ipaddress = node.attributes.get('ipaddress')
         
         print('INFO: Update node: %s, set IP: %s'%(node_id, ipaddress)) 
         db.nodes.update({'node_chef_id':node_id},{'$set': {'ipaddress':ipaddress}})
     
     print('INFO: Update IPv4 address END!') 
Example #40
0
 def get_node_attrs_from_chef(self):
     try:
         env_subnets_dict = {}
         node_attribute_dict = {}
         for organization in self.awshelper_obj.get_organizations():
             node_attribute_dict = defaultdict(dict)
             node_list = Node.list(self.api)
             for environment in self.awshelper_obj.get_environments(organization):
                 for region in self.awshelper_obj.get_regions():
                     vpc_list = self.awshelper_obj.get_vpc_in_region(region)
                     if vpc_list:
                         for vpc in self.awshelper_obj.get_vpc_in_region(region):
                             env_subnets_dict = self.awshelper_obj.get_env_subnets(organization, region, vpc)
             for node in node_list:
                 node_obj = Node(node, api=self.api)
                 node_split = self.ah_obj.split_string(node, ["."])
                 if node_split is None or len(node_split)<=1:
                     pass
                 else:
                     node_subnet = self.ah_obj.split_string(node, ['.'])[1]
                     for key_tuple, environment in env_subnets_dict.iteritems():
                         if node_subnet in key_tuple:
                             environment = env_subnets_dict[key_tuple]
                             attribute_list = node_obj.attributes
                             if 'ec2' in attribute_list:
                                 if 'instance_id' in node_obj.attributes.get_dotted('ec2'):
                                     instance_id = node_obj.attributes.get_dotted('ec2.instance_id')
                                     node_attribute_dict[instance_id]['node'] = node
                             if 'os' in attribute_list:
                                 node_attribute_dict[instance_id]['os']=node_obj['os']
                             if 'os_version' in attribute_list:
                                 node_attribute_dict[instance_id]['os_version'] = node_obj['os_version']
                             if 'platform' in attribute_list:
                                 node_attribute_dict[instance_id]['platform'] = node_obj['platform']
                             if 'platform_version' in attribute_list:
                                 node_attribute_dict[instance_id]['platform_version'] = node_obj['platform_version']
                             if 'uptime' in attribute_list:
                                 node_attribute_dict[instance_id]['uptime'] = node_obj['uptime']
                             if 'idletime' in attribute_list:
                                 node_attribute_dict[instance_id]['idletime'] = node_obj['idletime']
         return dict(node_attribute_dict)
     except Exception as exp_object:
         exc_type, exc_obj, exc_tb = sys.exc_info()
         self.ah_obj.print_exception("chef_helper.py", "get_node_attrs_from_chef1()", exp_object, exc_type, exc_obj, exc_tb)
         return {}
Example #41
0
 def destroy(self):
     cnode = CNode(self.name)
     if self['in_use'] == "provisioned":
         # Return to pool if the node is clean
         cnode.chef_environment = "_default"
         cnode.save()
     else:
         # Remove active model if the node is dirty
         active_model = cnode['razor_metadata']['razor_active_model_uuid']
         self.razor.remove_active_model(active_model)
         self.run_cmd("reboot 0")
         CClient(self.name).delete()
         cnode.delete()
         sleep(15)
Example #42
0
 def destroy(self):
     cnode = CNode(self.name)
     if self['in_use'] == "provisioned":
         # Return to pool if the node is clean
         cnode.chef_environment = "_default"
         cnode.save()
     else:
         # Remove active model if the node is dirty
         active_model = cnode['razor_metadata']['razor_active_model_uuid']
         self.razor.remove_active_model(active_model)
         self.run_cmd("reboot 0")
         CClient(self.name).delete()
         cnode.delete()
         sleep(15)
Example #43
0
    def get_machine_name(self):
        """
        Get the human readable machine name using the machine type and
        searching Knife for similar servers
        """
        # This must be called to get the correct Knife Setup for Node()
        # Even if we don't use the resulting API object directly
        api = ChefAPI.from_config_file(self.KNIFE_CONFIG)
        if self.VPC:
            base_name = "yip_%s%%s" % self.machine_type
        else:
            base_name = "yipit_%s%%s" % self.machine_type

        index = 1
        while True:
            name = base_name % index
            node = Node(name)
            if node.exists:
                index += 1
            else:
                break

        return name
def lambda_handler(event, context):

    ec2c = boto3.client('ec2')
    api = chef.autoconfigure()
    chef_nodes = {}
    aws_is_running = []
    nodes_to_remove = []

    #Array of running ec2 instances - ip addresses
    for region in ec2c.describe_regions()['Regions']:
        ec2 = boto3.resource('ec2', region_name=region['RegionName'])
        for instance in ec2.instances.filter(Filters=[{'Name': 'instance-state-name', 'Values': ['running']}]):
            #print('Debug EC2 Instances: ', region['RegionName'], instance.private_ip_address)
            aws_is_running.append(instance.private_ip_address)

    #Dictionary of ip addresses (key) and Node objects (value) from Chef
    for name, nodeobj in Node.list().iteritems() :
        for key, value in nodeobj.attributes.iteritems():
            if key == 'ipaddress' :
                #print('Debug Chef Nodes: ', value, nodeobj)
                chef_nodes.update({value : nodeobj})

    #Calculating nodes to remove
    for key, node in chef_nodes.iteritems():
        if key not in aws_is_running:
            nodes_to_remove.append(node)

    #Removing nodes in Chef that are no longer in AWS
    for node in nodes_to_remove:
        #print('Debug Nodes to Remove:', node['ipaddress'])
        node.delete()

    #So that the removed nodes are logged
    print 'Removed', [x['ipaddress'] for x in nodes_to_remove]

    return 'Removed', [x['ipaddress'] for x in nodes_to_remove]
Example #45
0
        for compute in computes:
            compute_ip = rpcsqa.set_node_in_use(compute, "agent")
            rpcsqa.remove_chef(compute)
            rpcsqa.install_opencenter(compute, results.repo, 'agent',
                                      oc_server_ip)

        # Print Cluster Info
        print "************************************************************"
        print "2 VMs, 1 controller ( VM Host ), %i Agents" % len(computes)
        print "OpenCenter Server (VM) with IP: %s on Host: %s" % (oc_server_ip,
                                                                  controller)
        print "Chef Server (VM) with IP: %s on Host: %s" % (chef_server_ip,
                                                            controller)
        print "Controller Node: %s with IP: %s" % (controller, controller_ip)
        for agent in computes:
            node = Node(agent)
            print "Agent Node: %s with IP: %s" % (agent, node['ipaddress'])
        print "************************************************************"

    else:
        #Pick an opencenter server, and rest for agents
        server = opencenter_list[0]
        dashboard = []
        clients = []
        if len(opencenter_list) > 1:
            dashboard = opencenter_list[1]
        if len(opencenter_list) > 2:
            agents = opencenter_list[2:]

        #Remove chef client...install opencenter server
        print "Making %s the server node" % server
Example #46
0
    def command(self):
        # Initialization
        logger.info("MIGRATION SCRIPT FOR FILES LIST POLICY")
        logger.info("######################################")

        # Disabling InsecureRequestWarning Unverified HTTPS request
        requests.packages.urllib3.disable_warnings()

        sanitized = False
        computers = set()
        self.api = _get_chef_api(self.settings.get('chef.url'),
                                 toChefUsername(self.options.chef_username),
                                 self.options.chef_pem, False,
                                 self.settings.get('chef.version'))

        self.auth_user = self.db.adminusers.find_one(
            {'username': self.options.chef_username})
        if self.auth_user is None:
            logger.error('The administrator user must exist in MongoDB')
            sys.exit(1)

        self.db = self.pyramid.db

        # Get local_file (File list) policy
        logger.info('Getting policy schema (local_file_res) ...')
        policy = self.db.policies.find_one({'slug': 'local_file_res'})
        schema = policy['schema']
        policyId = policy['_id']

        logger.info('schema   = %s' % str(schema))
        logger.debug('policyId = %s' % str(policyId))

        # Searching nodes with the File List policy
        # Query Fields of an Embedded Document (Mongo documentation)
        # Example:
        # db.nodes.find({"policies.58c8122a0dfd425b0894d5b6":{$exists:true}})
        logger.info('Searching for nodes with applied policy...')
        field = 'policies.' + str(policyId)
        filters = {field: {'$exists': True}}
        nodes = self.db.nodes.find(filters)

        # Validating data and, where appropiate, fixing
        for node in nodes:
            instance = node['policies'][unicode(policyId)]
            logger.debug("node = %s" % str(node))

            logger.info('-----------------------------------------------')
            logger.info('Node name = %s, mongo_id = %s' %
                        (node['name'], str(node['_id'])))
            logger.info('Instance of the policy on the node: %s' %
                        str(instance))
            while True:
                try:
                    validate(instance, schema)
                    break
                except ValidationError as e:
                    logger.warn('Validation error on instance: instance = %s' %
                                str(instance))
                    logger.warn(
                        'Validation error on instance: message error = %s' %
                        str(e.message))
                    # Sanitize instance
                    self.sanitize(e, instance)
                    sanitized = True

            if sanitized:
                # Setting false sanitized for next iteration
                sanitized = False
                logger.info(
                    'Sanitized instance of the policy on the node AFTER calling the validate method: %s'
                    % str(instance))

                # Update mongo
                logger.info('Updating instance in database (mongo) ...')
                self.db.nodes.update({'_id': node['_id']},
                                     {'$set': {
                                         field: instance
                                     }})

                logger.info('Recalculating policies in the node.')

                # Affected nodes
                if node['type'] == 'ou':
                    result = list(
                        self.db.nodes.find(
                            {
                                'path': get_filter_nodes_belonging_ou(
                                    node['_id']),
                                'type': 'computer'
                            }, {'_id': 1}))
                    logger.info('OU computers = %s' % str(result))
                elif node['type'] == 'group':
                    result = list(
                        self.db.nodes.find(
                            {
                                '_id': {
                                    '$in': node['members']
                                },
                                'type': 'computer'
                            }, {'_id': 1}))
                    logger.info('GROUP computers = %s' % str(result))
                elif node['type'] == 'computer':
                    result = [node]
                    logger.info('COMPUTER computers = %s' % str(result))

                [computers.add(str(n['_id'])) for n in result]

        for computer in computers:
            logger.info(
                'Applying policies to COMPUTER. For more information, see "gecosccui-celery.log" file'
            )
            computer = self.db.nodes.find_one({'_id': ObjectId(computer)})
            apply_policies_to_computer(self.db.nodes,
                                       computer,
                                       self.auth_user,
                                       api=self.api,
                                       initialize=False,
                                       use_celery=True)

        # Removing unused attributes (copy_files, delete_files) in chef nodes
        logger.info('\n')
        attrs = [
            "%s.copy_files" % (policy['path']),
            "%s.delete_files" % (policy['path'])
        ]
        logger.info('Removing unused attributes %s in chef nodes ...' % attrs)
        logger.info('\n')

        for node_id in ChefNode.list():
            node = ChefNode(node_id, self.api)
            logger.info('Checking node: %s' % (node_id))

            for attr in attrs:
                try:
                    if node.attributes.has_dotted(attr):
                        logger.warn("Remove %s attribute!" % attr)
                        delete_dotted(node.attributes, attr)

                    node.save()

                except:
                    logger.warn(
                        "Problem deleting attribute %s value from node: %s" %
                        (attr, node_id))
                    logger.warn(
                        "You may be trying to delete a default attribute instead normal attribute: %s"
                        % (node_id))

        logger.info('Finished.')
 def setUp(self):
     super(NodeTestCase, self).setUp()
     self.node = Node('test_1')
    def command(self):
        # Initialization
        sanitized = False
        computers = set()
        self.api = _get_chef_api(self.settings.get('chef.url'),
                                 toChefUsername(self.options.chef_username),
                                 self.options.chef_pem, False,
                                 self.settings.get('chef.version'))

        self.auth_user = self.db.adminusers.find_one(
            {'username': self.options.chef_username})
        if self.auth_user is None:
            logger.error('The administrator user must exist in MongoDB')
            sys.exit(1)

        self.db = self.pyramid.db

        # Get local_admin_users_res (Local Administrators) policy
        logger.info(
            'Getting Local Administrators (local_admin_users_res) policy ...')
        policy = self.db.policies.find_one({'slug': 'local_admin_users_res'})
        schema = policy['schema']
        policyId = policy['_id']

        logger.info('schema   = %s' % str(schema))
        logger.info('Id.policy = %s' % str(policyId))

        # Searching nodes with the Local Administrators policy
        # Query Fields of an Embedded Document (Mongo documentation)
        # Example:
        # db.nodes.find({"policies.58c8122a0dfd425b0894d5b6":{$exists:true}})
        logger.info('Searching nodes with the Local Administrators policy...')
        field = 'policies.' + str(policyId)
        filters = {field: {'$exists': True}}
        nodes = self.db.nodes.find(filters)

        # Validating data and, where appropiate, fixing
        for node in nodes:
            instance = node['policies'][unicode(policyId)]

            logger.info('Node name = %s, _id = %s' %
                        (node['name'], str(node['_id'])))
            logger.info('Instance before validate method: %s' % str(instance))
            while True:
                try:
                    validate(instance, schema)
                    break
                except ValidationError as e:
                    logger.warning('Validation error on instance = %s' %
                                   str(e.message))
                    # Sanitize instance
                    self.sanitize(e, instance)
                    sanitized = True

            if sanitized:
                # Setting false sanitized for next iteration
                sanitized = False
                logger.info('Sanitized instance: %s' % str(instance))

                # Update mongo
                self.db.nodes.update({'_id': node['_id']},
                                     {'$set': {
                                         field: instance
                                     }})

                # Affected nodes
                if node['type'] == 'ou':
                    result = list(
                        self.db.nodes.find(
                            {
                                'path': get_filter_nodes_belonging_ou(
                                    node['_id']),
                                'type': 'computer'
                            }, {'_id': 1}))
                    logger.info('OU computers = %s' % str(result))
                elif node['type'] == 'group':
                    result = list(
                        self.db.nodes.find(
                            {
                                '_id': {
                                    '$in': node['members']
                                },
                                'type': 'computer'
                            }, {'_id': 1}))
                    logger.info('GROUP computers = %s' % str(result))
                elif node['type'] == 'computer':
                    result = [node]
                    logger.info('COMPUTER computers = %s' % str(result))

                [computers.add(str(n['_id'])) for n in result]

        # Removing unused local_admin_remove_list attribute in chef nodes
        for node_id in ChefNode.list():
            node = ChefNode(node_id, self.api)
            logger.info('Checking node: %s' % (node_id))
            attr_dotted = policy['path'] + '.local_admin_remove_list'
            logger.info('Atttribute dotted path: %s' % (attr_dotted))
            if node.attributes.has_dotted(attr_dotted):
                logger.info("Remove 'local_admin_remove_list' attribute!")
                try:
                    logger.info(
                        "node.attributes = %s" %
                        str(node.attributes['gecos_ws_mgmt']['misc_mgmt']
                            ['local_admin_users_res'].to_dict()))
                    delete_dotted(node.attributes, attr_dotted)
                    node.save()
                except:
                    logger.warn(
                        "Problem deleting local_admin_remove_list value from node: %s"
                        % (node_id))
                    logger.warn(
                        "You may be trying to delete a default attribute instead normal attribute: %s"
                        % (node_id))

        for computer in computers:
            logger.info('computer = %s' % str(computer))
            computer = self.db.nodes.find_one({'_id': ObjectId(computer)})
            apply_policies_to_computer(self.db.nodes,
                                       computer,
                                       self.auth_user,
                                       api=self.api,
                                       initialize=False,
                                       use_celery=False)

        logger.info('Finished.')
Example #49
0
    def command(self):
        db = self.pyramid.db
        ou = db.nodes.find_one({'_id': ObjectId(self.options.ou_id)})
        if not ou:
            print 'Error OU does not exists'
            return
        comp = db.nodes.find_one({'_id': ObjectId(self.options.comp_id)})
        if not comp:
            print 'Error computer does not exists'
            return
        node_id = comp.get('node_chef_id', None)
        if not comp:
            print 'Error this computer has not node_chef_id'
            return
        policies = comp.get('policies', None)
        if policies != {}:
            print 'Error this computer should not have any policies'
            return
        admin = db.adminusers.find_one({'username': self.options.gcc_username})
        if not admin:
            print 'Error this admin does not exists'
            return
        elif not admin.get('is_superuser', None):
            print 'You need a super admin'
            return
        number_nodes = int(self.options.number)
        api = get_chef_api(self.settings,
                           admin)
        node = ChefNode(node_id, api)
        for i in range(number_nodes):
            new_node_name = '%s-%s' % (self.options.prefix, i)
            new_node = ChefNode(new_node_name, api)
            for attr in node.to_dict().keys():
                if hasattr(node, attr) and attr != 'name':
                    if attr == 'automatic':
                        automatic_dict = node.automatic.to_dict()
                        automatic_dict['ohai_gecos']['pclabel'] = new_node_name
                        user1 = 'user.name-%s-1' % new_node_name
                        user2 = 'user.name-%s-2' % new_node_name
                        automatic_dict['ohai_gecos']['users'] = [{'username': user1,
                                                                  'home': '/home/%s' % user1,
                                                                  'gid': 1000,
                                                                  'sudo': False,
                                                                  'uid': 1000},
                                                                 {'username': user2,
                                                                  'home': '/home/%s' % user2,
                                                                  'gid': 1000,
                                                                  'sudo': False,
                                                                  'uid': 1001}]

                        automatic = NodeAttributes(automatic_dict)
                        setattr(new_node, attr, automatic)
                    elif attr == 'normal':
                        node.normal.set_dotted('ohai_gecos', {})
                    else:
                        setattr(new_node, attr, getattr(node, attr))
            new_node.save()
            print 'Created %s at chef' % new_node_name
            res = requests.post('%s/register/computer/' % self.options.gcc_url,
                                {'ou_id': self.options.ou_id, 'node_id': new_node_name},
                                auth=(self.options.gcc_username, self.options.gcc_password))
            if res.ok and res.json()['ok']:
                print 'Created %s at gcc' % new_node_name
            elif res.ok and not res.json()['ok']:
                print 'Error %s at gcc' % new_node_name
                print '\t %s' % res.json()['message']
            else:
                print 'Unknow error %s at gcc' % new_node_name

            res = requests.put('%s/chef/status/' % self.options.gcc_url,
                               {'node_id': new_node_name,
                                'gcc_username': self.options.gcc_username})

            if res.ok and res.json()['ok']:
                print 'Chef client %s' % new_node_name
            elif res.ok and not res.json()['ok']:
                print 'Error %s at chef client' % new_node_name
                print '\t %s' % res.json()['message']
            else:
                print 'Unknow error %s at chef client' % new_node_name

        waiting_to_celery(db)
Example #50
0
    def command(self):
        from gecoscc.api.chef_status import USERS_OLD, USERS_OHAI
        # Initialization
        self.api = _get_chef_api(self.settings.get('chef.url'),
                            toChefUsername(self.options.chef_username),
                            self.options.chef_pem, False, self.settings.get('chef.version'))

        self.db = self.pyramid.db

        # Check administrator user
        auth_user = self.db.adminusers.find_one({'username': self.options.chef_username})
        if auth_user is None:
            logger.error('The administrator user must exist in MongoDB')
            sys.exit(1)

        # Recorriendo todos los nodos 
        for node_id in ChefNode.list():
            node = ChefNode(node_id, self.api)
            logger.info('Checking node: %s'%(node_id))
            
            try:
                if node.attributes.get_dotted(USERS_OLD):
                    delete_dotted(node.attributes, USERS_OLD)
                    node.save()
            except KeyError:
                logger.warn("Not found attribute: %s"%(USERS_OLD))
            except:
                logger.warn("Problem deleting users_old attribute from node: %s"%(node_id))

            # Updating users list    
            computer = self.db.nodes.find_one({'node_chef_id': node_id, 'type':'computer'})
            if not computer:
                logger.error('This node does not exist (mongodb)')
                continue
 
            chef_node_usernames = set([d['username'] for d in  node.attributes.get_dotted(USERS_OHAI)])
            gcc_node_usernames  = set([d['name'] for d in self.db.nodes.find({
                                       'type':'user',
                                       'computers': {'$in': [computer['_id']]}
                                    },
                                    {'_id':0, 'name':1})
                                 ])


            users_recalculate_policies = []
            users_remove_policies = []

            # Users added/removed ?
            if set.symmetric_difference(chef_node_usernames, gcc_node_usernames):
                logger.info("Users added/removed found.")

                # Add users or vinculate user to computer if already exists
                addusers = set.difference(chef_node_usernames, gcc_node_usernames)
                for add in addusers:
                    logger.info("Added user: %s"%(add))
                    user = self.db.nodes.find_one({'name': add, 'type': 'user', 'path': get_filter_in_domain(computer)})

                    if not user:
                        user_model = User()
                        user = user_model.serialize({'name': add,
                                                     'path': computer.get('path', ''),
                                                     'type': 'user',
                                                     'lock': computer.get('lock', ''),
                                                     'source': computer.get('source', '')})

                        user = update_computers_of_user(self.db, user, self.api)

                        del user['_id']
                        user_id = self.db.nodes.insert(user)
                        user = self.db.nodes.find_one({'_id': user_id})
                        users_recalculate_policies.append(user)

                    else:
                        computers = user.get('computers', [])
                        if computer['_id'] not in computers:
                            computers.append(computer['_id'])
                            self.db.nodes.update({'_id': user['_id']}, {'$set': {'computers': computers}})
                            users_recalculate_policies.append(user)
                            add_computer_to_user(computer['_id'], user['_id'])

                # Removed users
                delusers = set.difference(gcc_node_usernames, chef_node_usernames)
                for delete in delusers:
                    logger.info("Deleted user: %s"%(delete))
                    user = self.db.nodes.find_one({'name': delete,
                                                   'type': 'user',
                                                   'path': get_filter_in_domain(computer)})
                    computers = user['computers'] if user else []
                    if computer['_id'] in computers:
                        users_remove_policies.append(deepcopy(user))
                        computers.remove(computer['_id'])
                        self.db.nodes.update({'_id': user['_id']}, {'$set': {'computers': computers}})

                for user in users_recalculate_policies:
                    apply_policies_to_user(self.db.nodes, user, auth_user)

                for user in users_remove_policies:
                    remove_policies_of_computer(user, computer, auth_user)
 def get_all_nodes(self):
     return Node.list().names
    def command(self):
        # Initialization
        logger.info("MIGRATION SCRIPT FOR USER_APPS_AUTOSTART POLICY")
        logger.info("###############################################")

        # Disabling InsecureRequestWarning Unverified HTTPS request
        requests.packages.urllib3.disable_warnings()

        sanitized = False
        ous = []
        groups = []
        users = []
        self.api = _get_chef_api(self.settings.get('chef.url'),
                                 toChefUsername(self.options.chef_username),
                                 self.options.chef_pem, False,
                                 self.settings.get('chef.version'))

        self.auth_user = self.db.adminusers.find_one(
            {'username': self.options.chef_username})
        if self.auth_user is None:
            logger.error('The administrator user must exist in MongoDB')
            sys.exit(1)

        self.db = self.pyramid.db

        # Get local_users (Users) policy
        logger.info('Getting policy schema (user_apps_autostart_res) ...')
        policy = self.db.policies.find_one({'slug': 'user_apps_autostart_res'})
        schema = policy['schema']
        policyId = policy['_id']

        logger.info('schema   = %s' % str(schema))
        logger.debug('policyId = %s' % str(policyId))

        # Searching nodes with the Local Administrators policy
        # Query Fields of an Embedded Document (Mongo documentation)
        # Example:
        # db.nodes.find({"policies.58c8122a0dfd425b0894d5b6":{$exists:true}})
        logger.info('Searching for nodes with applied policy...')
        field = 'policies.' + str(policyId)
        filters = {field: {'$exists': True}}
        nodes = self.db.nodes.find(filters)

        # Validating data and, where appropiate, fixing
        for node in nodes:
            instance = node['policies'][unicode(policyId)]
            logger.info("node = %s" % str(node))

            logger.info('-----------------------------------------------')
            logger.info('Node name = %s, mongo_id = %s' %
                        (node['name'], str(node['_id'])))
            logger.info('Instance of the policy on the node: %s' %
                        str(instance))
            while True:
                try:
                    validate(instance, schema)
                    break
                except ValidationError as e:
                    logger.warn('Validation error on instance = %s' %
                                str(e.message))
                    # Sanitize instance
                    self.sanitize(e, instance)
                    sanitized = True

            if sanitized:
                # Setting false sanitized for next iteration
                sanitized = False
                logger.info(
                    'Sanitized instance of the policy on the node AFTER calling the validate method: %s'
                    % str(instance))

                # Update mongo
                logger.info('Updating instance in database (mongo) ...')
                self.db.nodes.update({'_id': node['_id']},
                                     {'$set': {
                                         field: instance
                                     }})

                logger.info('Recalculating policies in the node.')
                # Affected nodes
                if node['type'] == 'ou':
                    ous.append(node)
                elif node['type'] == 'group':
                    groups.append(node)
                elif node['type'] == 'user':
                    users.append(node)

        # We only go through the highest level OUs.
        # Therefore, we eliminate intermediate OUs and
        # then do not recalculate the policies
        # for the same node several times.
        for ou in ous:
            parents = [
                ObjectId(oid) for oid in ou['path'].split(',') if oid != 'root'
            ]
            if any(o['_id'] in parents for o in ous):
                ous.remove(ou)

        # Users that are not under an OU or GROUP that have the migrated policy
        for user in users:
            parents = [
                ObjectId(oid) for oid in user['path'].split(',')
                if oid != 'root'
            ]
            if any(o['_id'] in parents for o in ous):
                users.remove(user)
            elif any(user['_id'] in group['members'] for group in groups):
                users.remove(user)

        # Recalculating policies for OU
        for ou in ous:
            old = deepcopy(ou)
            del old["policies"][str(policyId)]
            object_changed(self.auth_user, 'ou', ou, old)

        # Recalculating policies for GROUP
        for group in groups:
            old = deepcopy(group)
            del old["policies"][str(policyId)]
            object_changed(self.auth_user, 'group', group, old)

        # Recalculating policies for USER
        for user in users:
            old = deepcopy(user)
            del old["policies"][str(policyId)]
            object_changed(self.auth_user, 'user', user, old)

        # Removing unused desktops_to_remove attribute in chef nodes
        logger.info('\n')
        logger.info(
            'Removing unused desktops_to_remove attribute in chef nodes ...')
        for node_id in ChefNode.list():
            node = ChefNode(node_id, self.api)
            logger.info('Checking node: %s' % (node_id))
            field_chef = '%s.users' % policy['path']
            users = node.attributes.get_dotted(
                field_chef) if node.attributes.has_dotted(field_chef) else []
            for user in users:
                logger.debug("user = %s" % (user))
                attr_delete_path = '%s.%s.desktops_to_remove' % (field_chef,
                                                                 user)
                logger.debug('Atttribute dotted path: %s' % (attr_delete_path))
                if node.attributes.has_dotted(attr_delete_path):
                    logger.warn(
                        "Remove 'desktops_to_remove' attribute! for user %s" %
                        (user))
                    try:
                        delete_dotted(node.attributes, attr_delete_path)
                        node.save()
                    except:
                        logger.warn(
                            "Problem deleting desktops_to_remove value from node: %s"
                            % (node_id))
                        logger.warn(
                            "You may be trying to delete a default attribute instead normal attribute: %s"
                            % (node_id))

        logger.info('Finished.')
Example #53
0
#!/usr/bin/env python
#
# requires
#   - pip install PyChef
#   - access to knife.rb file
#   - access to client.pem file
#
# DON'T RUN THIS - THESE ARE JUST EXAMPLES

#------------------------------
# update node info
from chef import autoconfigure, Node

api = autoconfigure()
n = Node('web1')
print n['fqdn']
n['myapp']['version'] = '1.0'
n.save()
#------------------------------

#------------------------------
# get client list
from chef import autoconfigure

api = autoconfigure()
print api.api_request('GET', '/clients')
#------------------------------
Example #54
0
def save_node_and_free(node, api=None, refresh=False):
    if refresh and api:
        node = ChefNode(node.name, api)
    node.attributes.set_dotted(USE_NODE, {})
    node.save()
    def command(self):
        # Initialization
        self.api = _get_chef_api(self.settings.get('chef.url'),
                            toChefUsername(self.options.chef_username),
                            self.options.chef_pem, False, self.settings.get('chef.version'))

        self.auth_user = self.db.adminusers.find_one({'username': self.options.chef_username})       
        if self.auth_user is None:
            logger.error('The administrator user must exist in MongoDB')
            return
        

        self.db = self.pyramid.db
        ous = []
        groups = []
        computers = []

        # Get packages policy ID
        packages_policy = self.db.policies.find_one({"slug": "package_res"})
        if packages_policy is None:
            logger.error('Can\'t detect "package_res" policy!')
            return
            
        if 'pkgs_to_remove' in packages_policy["schema"]["properties"]:
            logger.error("The 'package_res' policy in the system is deprecated, please update to new package policy!")
            return
        
            
        logger.info('Packages policy ID: %s'%(str(packages_policy['_id'])))
        
        # Get all nodes with old package policy data
        logger.info('Getting all nodes with old package policy data...')
        path_to_find = "policies.%s.pkgs_to_remove"%(str(packages_policy['_id']))
        old_policy_nodes = self.db.nodes.find({ '$query': {path_to_find: { '$exists': True }}, '$orderby': { "path" : 1 }})
        
        updated_nodes = []
        for node in old_policy_nodes:
            logger.info('Updating node %s ...'%(str(node['_id'])))
            updated_nodes.append(str(node['_id']))
            
            logger.debug('Packages to add: %s'%(str(node['policies'][str(packages_policy['_id'])]['package_list'])))
            logger.debug('Packages to remove: %s'%(str(node['policies'][str(packages_policy['_id'])]['pkgs_to_remove'])))
            
            # Join the lists
            package_list = []
            for package_name in node['policies'][str(packages_policy['_id'])]['package_list']:
                package_list.append({'name': package_name, 'version': 'current', 'action': 'add'})
            
            for package_name in node['policies'][str(packages_policy['_id'])]['pkgs_to_remove']:
                package_list.append({'name': package_name, 'version': 'current', 'action': 'remove'})
            
            if 'pkgs_to_remove' in node['policies'][str(packages_policy['_id'])]:
                del node['policies'][str(packages_policy['_id'])]['pkgs_to_remove']
                
            node['policies'][str(packages_policy['_id'])]['package_list'] = package_list
            
            # Update policies
            self.db.nodes.update({'_id': node['_id']}, {'$set': {'policies': node['policies']}})
            logger.debug('Joined list: %s'%(str(node['policies'][str(packages_policy['_id'])]['package_list'])))

            if   node['type'] == 'ou':
                ous.append(node)                                    
            elif node['type'] == 'group':                                             
                groups.append(node)
            elif node['type'] == 'computer':
                computers.append(node)

        logger.info('%s nodes were updated!'%(len(updated_nodes)))

        # We only go through the highest level OUs. 
        # Therefore, we eliminate intermediate OUs and 
        # then do not recalculate the policies
        # for the same node several times.
        for ou in ous:
            parents = [ObjectId(oid) for oid in ou['path'].split(',') if oid != 'root']
            if any(o['_id'] in parents for o in ous):
                ous.remove(ou)

        # Users that are not under an OU or GROUP that have the migrated policy
        for computer in computers:
            parents = [ObjectId(oid) for oid in computer['path'].split(',') if oid != 'root'] 
            if any(o['_id'] in parents for o in ous):
                computers.remove(computer)
            elif any(computer['_id'] in group['members'] for group in groups):
                computers.remove(computer)

        # Recalculating policies for OU
        for ou in ous: 
            old = deepcopy(ou)
            del old["policies"][str(packages_policy['_id'])]
            object_changed(self.auth_user, 'ou', ou, old)

        # Recalculating policies for GROUP
        for group in groups:
            old = deepcopy(group)
            del old["policies"][str(packages_policy['_id'])]
            object_changed(self.auth_user, 'group', group, old)

        # Recalculating policies for USER
        for computer in computers:
            old = deepcopy(computer)
            del old["policies"][str(packages_policy['_id'])]
            object_changed(self.auth_user, 'computer', computer, old)

        # Recalculate policies for Chef nodes
        for node_id in ChefNode.list():
            node = ChefNode(node_id, self.api)
            logger.info('Checking node: %s'%(node_id))
            if ("gecos_ws_mgmt" in node.attributes) and ("software_mgmt" in node.attributes["gecos_ws_mgmt"]) and ("package_res" in node.attributes["gecos_ws_mgmt"]["software_mgmt"]):
                if "pkgs_to_remove" in node.attributes["gecos_ws_mgmt"]["software_mgmt"]["package_res"]:
                    logger.debug("Chef node %s contains a pkgs_to_remove value!"%(node_id))
                    # Remove pkgs_to_remove from mongodb node
                    logger.info("Remove 'pkgs_to_remove' attribute!")
                    try:
                        del node.attributes["gecos_ws_mgmt"]["software_mgmt"]["package_res"]["pkgs_to_remove"]
                        node.save()
                    except:
                        logger.warn("Problem deleting pkgs_to_remove value from node: %s"%(node_id))
                    
                if not "package_list" in node.attributes["gecos_ws_mgmt"]["software_mgmt"]["package_res"]:
                    logger.error("Chef node %s doesn\'t contains a package_list value!"%(node_id))
                    continue
                
                package_list = node.attributes["gecos_ws_mgmt"]["software_mgmt"]["package_res"]["package_list"]
                for element in package_list:
                    if not 'action' in element:
                        logger.debug('Chef node: %s doesn\'t have an action value in package_res! (package_list:%s)'%(node_id, str(package_list))) 
                        break

        # Final check
        bad_nodes = Search('node', "pkgs_to_remove:*", rows=1000, start=0, api=self.api)
        for node in bad_nodes:
            logger.warn('Detected bad node: %s'%(node.object.name))
            gecos_node = self.db.nodes.find_one({"node_chef_id": node.object.name})
            if gecos_node is None:
                logger.warn('Can\'t find node in MongoDB for: %s'%(node.object.name)) 
            else:
                logger.warn('For an unknown reason a computer called %s wasn\'t updated!'%(gecos_node['name'])) 
                
        logger.info('END ;)')