def put(self):
node_id = self.request.POST.get('node_id')
username = self.request.POST.get('gcc_username')
if not node_id:
return {'ok': False,
'message': 'Please set a node id (node_id)'}
if not username:
return {'ok': False,
'message': 'Please set a admin username (gcc_username)'}
self.request.user = self.request.db.adminusers.find_one({'username': username})
if not self.request.user:
return {'ok': False,
'message': 'The admin user %s does not exists' % username}
settings = get_current_registry().settings
api = get_chef_api(settings, self.request.user)
node = Node(node_id, api)
job_status = node.attributes.get('job_status')
reserve_node = False
if job_status:
node = reserve_node_or_raise(node_id, api, 'gcc-chef-status-%s' % random.random(), attempts=3)
reserve_node = True
chef_client_error = False
for job_id, job_status in job_status.to_dict().items():
job = self.collection.find_one({'_id': ObjectId(job_id)})
if not job:
continue
if job_status['status'] == 0:
self.collection.update({'_id': job['_id']},
{'$set': {'status': 'finished',
'last_update': datetime.datetime.utcnow()}})
else:
chef_client_error = True
self.collection.update({'_id': job['_id']},
{'$set': {'status': 'errors',
'message': job_status.get('message', 'Error'),
'last_update': datetime.datetime.utcnow()}})
self.request.db.nodes.update({'node_chef_id': node_id}, {'$set': {'error_last_chef_client': chef_client_error}})
invalidate_jobs(self.request)
node.attributes.set_dotted('job_status', {})
users_old = self.get_attr(node, USERS_OLD)
users = self.get_attr(node, USERS_OHAI)
if not users_old or users_old != users:
if not reserve_node:
node = reserve_node_or_raise(node_id, api, 'gcc-chef-status-%s' % random.random(), attempts=3)
return self.check_users(node)
if job_status:
save_node_and_free(node)
return {'ok': True}
def object_action(self, user, obj, objold=None, action=None, computers=None):
api = get_chef_api(self.app.conf, user)
cookbook = get_cookbook(api, self.app.conf.get('chef.cookbook_name'))
computers = computers or self.get_related_computers(obj)
are_new_jobs = False
for computer in computers:
try:
job_ids_by_computer = []
node_chef_id = computer.get('node_chef_id', None)
node = reserve_node_or_raise(node_chef_id, api, 'gcc-tasks-%s-%s' % (obj['_id'], random.random()), 10)
if not node.get(self.app.conf.get('chef.cookbook_name')):
raise NodeNotLinked("Node %s is not linked" % node_chef_id)
error_last_saved = computer.get('error_last_saved', False)
error_last_chef_client = computer.get('error_last_chef_client', False)
force_update = error_last_saved or error_last_chef_client
node, updated = self.update_node(user, computer, obj, objold, node, action, job_ids_by_computer, force_update)
if not updated:
save_node_and_free(node)
continue
are_new_jobs = True
self.validate_data(node, cookbook, api)
save_node_and_free(node)
if error_last_saved:
self.db.nodes.update({'_id': computer['_id']},
{'$set': {'error_last_saved': False}})
except NodeNotLinked as e:
self.report_node_not_linked(computer, user, obj, action)
are_new_jobs = True
save_node_and_free(node, api, refresh=True)
except NodeBusyException as e:
self.report_node_busy(computer, user, obj, action)
are_new_jobs = True
except ValidationError as e:
if not job_ids_by_computer:
self.report_unknown_error(e, user, obj, action, computer)
self.report_error(e, job_ids_by_computer, computer, 'Validation error: ')
save_node_and_free(node, api, refresh=True)
are_new_jobs = True
except Exception as e:
if not job_ids_by_computer:
self.report_unknown_error(e, user, obj, action, computer)
self.report_error(e, job_ids_by_computer, computer)
try:
save_node_and_free(node, api, refresh=True)
except:
pass
are_new_jobs = True
if are_new_jobs:
invalidate_jobs(self.request, user)
def object_action(self,
user,
obj,
objold=None,
action=None,
computers=None):
api = get_chef_api(self.app.conf, user)
cookbook = get_cookbook(api, self.app.conf.get('chef.cookbook_name'))
computers = computers or self.get_related_computers(obj)
are_new_jobs = False
for computer in computers:
try:
job_ids_by_computer = []
node_chef_id = computer.get('node_chef_id', None)
node = reserve_node_or_raise(
node_chef_id, api,
'gcc-tasks-%s-%s' % (obj['_id'], random.random()), 10)
if not node.get(self.app.conf.get('chef.cookbook_name')):
raise NodeNotLinked("Node %s is not linked" % node_chef_id)
error_last_saved = computer.get('error_last_saved', False)
error_last_chef_client = computer.get('error_last_chef_client',
False)
force_update = error_last_saved or error_last_chef_client
node, updated = self.update_node(user, computer, obj, objold,
node, action,
job_ids_by_computer,
force_update)
if not updated:
save_node_and_free(node)
continue
are_new_jobs = True
self.validate_data(node, cookbook, api)
save_node_and_free(node)
if error_last_saved:
self.db.nodes.update({'_id': computer['_id']},
{'$set': {
'error_last_saved': False
}})
except NodeNotLinked as e:
self.report_node_not_linked(computer, user, obj, action)
are_new_jobs = True
save_node_and_free(node, api, refresh=True)
except NodeBusyException as e:
self.report_node_busy(computer, user, obj, action)
are_new_jobs = True
except ValidationError as e:
if not job_ids_by_computer:
self.report_unknown_error(e, user, obj, action, computer)
self.report_error(e, job_ids_by_computer, computer,
'Validation error: ')
save_node_and_free(node, api, refresh=True)
are_new_jobs = True
except Exception as e:
if not job_ids_by_computer:
self.report_unknown_error(e, user, obj, action, computer)
self.report_error(e, job_ids_by_computer, computer)
try:
save_node_and_free(node, api, refresh=True)
except:
pass
are_new_jobs = True
if are_new_jobs:
invalidate_jobs(self.request, user)
def post(self):
try:
# Initialize report
report = {'inserted': 0,
'updated': 0,
'total': 0,
'warnings': {}}
objects_apply_policy = {'computer': [],
'user': []}
db = self.request.db
# Read GZIP data
postedfile = self.request.POST['media'].file
xmldata = GzipFile('', 'r', 9, StringIO(postedfile.read())).read()
# Read XML data
xmldoc = minidom.parseString(xmldata)
# Get the root domain
xmlDomain = xmldoc.getElementsByTagName('Domain')[0]
is_ad_master = self.request.POST['master'] == 'True'
domain = self._get_domain(self.importSchema[0], xmlDomain, is_ad_master, report)
# Convert from AD objects to MongoDB objects
mongoObjects = {}
mongoObjectsPath = {}
for objSchema in self.importSchema:
objs = xmldoc.getElementsByTagName(objSchema['adName'])
for adObj in objs:
if not adObj.hasAttribute('ObjectGUID'):
raise Exception('An Active Directory object must has "ObjectGUID" attrib.')
mongoObject, is_saving = self._convertADObjectToMongoObject(domain, mongoObjects, objSchema, adObj, is_ad_master, report, objects_apply_policy)
report['total'] += 1
if is_saving:
mongoObjects[mongoObject['adDistinguishedName']] = mongoObject
mongoObjectsPath[mongoObject['adDistinguishedName']] = mongoObject
# Order mongoObjects by dependences
if mongoObjects:
mongoObjects[domain['adDistinguishedName']] = domain
mongoObjectsPath[domain['adDistinguishedName']] = domain
mongoObjects = self._orderByDependencesMongoObjects(mongoObjects, domain)
# Save each MongoDB objects
properRootDomainADDN = domain['adDistinguishedName']
for index, mongoObject in mongoObjects.items():
if index == properRootDomainADDN:
continue
# Get the proper path ("root,{0}._id,{1}._id,{2}._id...")
listPath = re.findall(ur'([^, ]+=(?:(?:\\,)|[^,])+)', index)
nodePath = ','.join(listPath[1:])
# Find parent
mongoObjectParent = mongoObjectsPath[nodePath]
mongoObjectParent = self.collection.find_one({'_id': mongoObjectParent['_id']})
path = '{0},{1}'.format(mongoObjectParent['path'], str(mongoObjectParent['_id']))
mongoObject['path'] = path
# Save mongoObject
self._saveMongoObject(mongoObject)
# AD Fixes
admin_user = self.request.user
chef_server_api = get_chef_api(get_current_registry().settings, admin_user)
if is_ad_master:
for index, mongoObject in mongoObjects.items():
if mongoObject['type'] == 'group':
if mongoObject['members'] != []:
mongoObject['members'] = []
self._saveMongoObject(mongoObject)
for index, mongoObject in mongoObjects.items():
updateMongoObject = False
# MemberOf
if mongoObject['type'] in ('user', 'computer'):
if 'memberof' not in mongoObject or is_ad_master:
mongoObject['memberof'] = []
if 'adPrimaryGroup' in mongoObject and mongoObject['adPrimaryGroup']:
group = mongoObjects[mongoObject['adPrimaryGroup']]
if is_visible_group(db, group['_id'], mongoObject):
if not mongoObject['_id'] in group['members']:
group['members'].append(mongoObject['_id'])
self._saveMongoObject(group)
if mongoObjects[mongoObject['adPrimaryGroup']]['_id'] not in mongoObject['memberof']:
mongoObject['memberof'].append(mongoObjects[mongoObject['adPrimaryGroup']]['_id'])
else:
self._warningGroup(group, mongoObject, report)
updateMongoObject = True
del mongoObject['adPrimaryGroup']
if 'adMemberOf' in mongoObject and mongoObject['adMemberOf']:
for group_id in mongoObject['adMemberOf']:
group = mongoObjects[group_id]
if is_visible_group(db, group['_id'], mongoObject):
if not mongoObject['_id'] in group['members']:
group['members'].append(mongoObject['_id'])
self._saveMongoObject(group)
if mongoObjects[group_id]['_id'] not in mongoObject['memberof']:
mongoObject['memberof'].append(mongoObjects[group_id]['_id'])
else:
self._warningGroup(group, mongoObject, report)
updateMongoObject = True
del mongoObject['adMemberOf']
# Create Chef-Server Nodes
if mongoObject['type'] == 'computer':
chef_server_node = reserve_node_or_raise(mongoObject['name'],
chef_server_api,
'gcc-ad-import-%s' % random.random(),
attempts=3)
ohai_gecos_in_runlist = self.RECIPE_NAME_OHAI_GECOS in chef_server_node.run_list
gecos_ws_mgmt_in_runlist = self.RECIPE_NAME_GECOS_WS_MGMT in chef_server_node.run_list
if not ohai_gecos_in_runlist and not gecos_ws_mgmt_in_runlist:
chef_server_node.run_list.append(self.RECIPE_NAME_OHAI_GECOS)
chef_server_node.run_list.append(self.RECIPE_NAME_GECOS_WS_MGMT)
elif not ohai_gecos_in_runlist and gecos_ws_mgmt_in_runlist:
chef_server_node.run_list.insert(chef_server_node.run_list.index(self.RECIPE_NAME_GECOS_WS_MGMT), self.RECIPE_NAME_OHAI_GECOS)
elif ohai_gecos_in_runlist and not gecos_ws_mgmt_in_runlist:
chef_server_node.run_list.insert(chef_server_node.run_list.index(self.RECIPE_NAME_OHAI_GECOS) + 1, self.RECIPE_NAME_GECOS_WS_MGMT)
save_node_and_free(chef_server_node)
chef_server_client = Client(mongoObject['name'], api=chef_server_api)
if not chef_server_client.exists:
chef_server_client.save()
mongoObject['node_chef_id'] = mongoObject['name']
updateMongoObject = True
# Save changes
if updateMongoObject:
self._saveMongoObject(mongoObject)
# apply policies to new objects
for node_type, node_names in objects_apply_policy.items():
nodes = self.collection.find({'name': {'$in': node_names},
'path': get_filter_this_domain(domain),
'type': node_type})
for node in nodes:
apply_policies_function = globals()['apply_policies_to_%s' % node['type']]
apply_policies_function(self.collection, node, admin_user, api=chef_server_api)
# Return result
status = '{0} inserted, {1} updated of {2} objects imported successfully.'.format(report['inserted'],
report['updated'],
report['total'])
response = {'status': status,
'ok': True}
except Exception as e:
logger.exception(e)
response = {'status': u'{0}'.format(e),
'ok': False}
warnings = report.get('warnings', [])
if warnings:
response['warnings'] = warnings
return response
def put(self):
node_id = self.request.POST.get('node_id')
username = self.request.POST.get('gcc_username')
if not node_id:
return {'ok': False,
'message': 'Please set a node id (node_id)'}
if not username:
return {'ok': False,
'message': 'Please set a admin username (gcc_username)'}
self.request.user = self.request.db.adminusers.find_one({'username': username})
if not self.request.user:
return {'ok': False,
'message': 'The admin user %s does not exists' % username}
settings = get_current_registry().settings
api = get_chef_api(settings, self.request.user)
node = Node(node_id, api)
job_status = node.attributes.get('job_status')
# After chef-client run, a report handler calls /api/chef_status
# Previously, gcc_link attribute of chef node is updated by network policies
gcc_link = node.attributes.get('gcc_link')
self.request.db.nodes.update({'node_chef_id':node_id},{'$set': {'gcc_link':gcc_link}})
reserve_node = False
if job_status:
node = reserve_node_or_raise(node_id, api, 'gcc-chef-status-%s' % random.random(), attempts=3)
reserve_node = True
chef_client_error = False
for job_id, job_status in job_status.to_dict().items():
job = self.collection.find_one({'_id': ObjectId(job_id)})
if not job:
continue
# Parent
macrojob = self.collection.find_one({'_id': ObjectId(job['parent'])}) if 'parent' in job else None
if job_status['status'] == 0:
self.collection.update({'_id': job['_id']},
{'$set': {'status': 'finished',
'last_update': datetime.datetime.utcnow()}})
# Decrement number of children in parent
if macrojob and 'counter' in macrojob:
macrojob['counter'] -= 1
elif job_status['status'] == 2:
self.collection.update({'_id': job['_id']},
{'$set': {'status': 'warnings',
'message': job_status.get('message', 'Warning'),
'last_update': datetime.datetime.utcnow()}})
if macrojob:
macrojob['status'] = 'warnings'
else:
chef_client_error = True
self.collection.update({'_id': job['_id']},
{'$set': {'status': 'errors',
'message': job_status.get('message', 'Error'),
'last_update': datetime.datetime.utcnow()}})
if macrojob:
macrojob['status'] = 'errors'
# Update parent
if macrojob:
self.collection.update({'_id': macrojob['_id']},
{'$set': {'counter': macrojob['counter'],
'message': self._("Pending: %d") % macrojob['counter'],
'status': 'finished' if macrojob['counter'] == 0 else macrojob['status']}})
self.request.db.nodes.update({'node_chef_id': node_id}, {'$set': {'error_last_chef_client': chef_client_error}})
invalidate_jobs(self.request)
node.attributes.set_dotted('job_status', {})
users_old = self.get_attr(node, USERS_OLD)
users = self.get_attr(node, USERS_OHAI)
if not users_old or users_old != users:
if not reserve_node:
node = reserve_node_or_raise(node_id, api, 'gcc-chef-status-%s' % random.random(), attempts=3)
return self.check_users(node, api)
if job_status:
save_node_and_free(node)
return {'ok': True}
def post(self):
try:
# Initialize report
report = {'inserted': 0, 'updated': 0, 'total': 0, 'warnings': {}}
objects_apply_policy = {'computer': [], 'user': []}
db = self.request.db
# Read GZIP data
postedfile = self.request.POST['media'].file
xmldata = GzipFile('', 'r', 9, StringIO(postedfile.read())).read()
# Read XML data
xmldoc = minidom.parseString(xmldata)
# Get the root domain
xmlDomain = xmldoc.getElementsByTagName('Domain')[0]
is_ad_master = self.request.POST['master'] == 'True'
domain = self._get_domain(self.importSchema[0], xmlDomain,
is_ad_master, report)
# Convert from AD objects to MongoDB objects
mongoObjects = {}
mongoObjectsPath = {}
for objSchema in self.importSchema:
objs = xmldoc.getElementsByTagName(objSchema['adName'])
for adObj in objs:
if not adObj.hasAttribute('ObjectGUID'):
raise Exception(
'An Active Directory object must has "ObjectGUID" attrib.'
)
mongoObject, is_saving = self._convertADObjectToMongoObject(
domain, mongoObjects, objSchema, adObj, is_ad_master,
report, objects_apply_policy)
report['total'] += 1
if is_saving:
mongoObjects[
mongoObject['adDistinguishedName']] = mongoObject
mongoObjectsPath[
mongoObject['adDistinguishedName']] = mongoObject
# Order mongoObjects by dependences
if mongoObjects:
mongoObjects[domain['adDistinguishedName']] = domain
mongoObjectsPath[domain['adDistinguishedName']] = domain
mongoObjects = self._orderByDependencesMongoObjects(
mongoObjects, domain)
# Save each MongoDB objects
properRootDomainADDN = domain['adDistinguishedName']
for index, mongoObject in mongoObjects.items():
if index == properRootDomainADDN:
continue
# Get the proper path ("root,{0}._id,{1}._id,{2}._id...")
listPath = re.findall(ur'([^, ]+=(?:(?:\\,)|[^,])+)', index)
nodePath = ','.join(listPath[1:])
# Find parent
mongoObjectParent = mongoObjectsPath[nodePath]
mongoObjectParent = self.collection.find_one(
{'_id': mongoObjectParent['_id']})
path = '{0},{1}'.format(mongoObjectParent['path'],
str(mongoObjectParent['_id']))
mongoObject['path'] = path
# Save mongoObject
self._saveMongoObject(mongoObject)
# AD Fixes
admin_user = self.request.user
chef_server_api = get_chef_api(get_current_registry().settings,
admin_user)
if is_ad_master:
for index, mongoObject in mongoObjects.items():
if mongoObject['type'] == 'group':
if mongoObject['members'] != []:
mongoObject['members'] = []
self._saveMongoObject(mongoObject)
for index, mongoObject in mongoObjects.items():
updateMongoObject = False
# MemberOf
if mongoObject['type'] in ('user', 'computer'):
if 'memberof' not in mongoObject or is_ad_master:
mongoObject['memberof'] = []
if 'adPrimaryGroup' in mongoObject and mongoObject[
'adPrimaryGroup']:
group = mongoObjects[mongoObject['adPrimaryGroup']]
if is_visible_group(db, group['_id'], mongoObject):
if not mongoObject['_id'] in group['members']:
group['members'].append(mongoObject['_id'])
self._saveMongoObject(group)
if mongoObjects[mongoObject['adPrimaryGroup']][
'_id'] not in mongoObject['memberof']:
mongoObject['memberof'].append(mongoObjects[
mongoObject['adPrimaryGroup']]['_id'])
else:
self._warningGroup(group, mongoObject, report)
updateMongoObject = True
del mongoObject['adPrimaryGroup']
if 'adMemberOf' in mongoObject and mongoObject[
'adMemberOf']:
for group_id in mongoObject['adMemberOf']:
group = mongoObjects[group_id]
if is_visible_group(db, group['_id'], mongoObject):
if not mongoObject['_id'] in group['members']:
group['members'].append(mongoObject['_id'])
self._saveMongoObject(group)
if mongoObjects[group_id][
'_id'] not in mongoObject['memberof']:
mongoObject['memberof'].append(
mongoObjects[group_id]['_id'])
else:
self._warningGroup(group, mongoObject, report)
updateMongoObject = True
del mongoObject['adMemberOf']
# Create Chef-Server Nodes
if mongoObject['type'] == 'computer':
chef_server_node = reserve_node_or_raise(
mongoObject['name'],
chef_server_api,
'gcc-ad-import-%s' % random.random(),
attempts=3)
ohai_gecos_in_runlist = self.RECIPE_NAME_OHAI_GECOS in chef_server_node.run_list
gecos_ws_mgmt_in_runlist = self.RECIPE_NAME_GECOS_WS_MGMT in chef_server_node.run_list
if not ohai_gecos_in_runlist and not gecos_ws_mgmt_in_runlist:
chef_server_node.run_list.append(
self.RECIPE_NAME_OHAI_GECOS)
chef_server_node.run_list.append(
self.RECIPE_NAME_GECOS_WS_MGMT)
elif not ohai_gecos_in_runlist and gecos_ws_mgmt_in_runlist:
chef_server_node.run_list.insert(
chef_server_node.run_list.index(
self.RECIPE_NAME_GECOS_WS_MGMT),
self.RECIPE_NAME_OHAI_GECOS)
elif ohai_gecos_in_runlist and not gecos_ws_mgmt_in_runlist:
chef_server_node.run_list.insert(
chef_server_node.run_list.index(
self.RECIPE_NAME_OHAI_GECOS) + 1,
self.RECIPE_NAME_GECOS_WS_MGMT)
save_node_and_free(chef_server_node)
chef_server_client = Client(mongoObject['name'],
api=chef_server_api)
if not chef_server_client.exists:
chef_server_client.save()
mongoObject['node_chef_id'] = mongoObject['name']
updateMongoObject = True
# Save changes
if updateMongoObject:
self._saveMongoObject(mongoObject)
# apply policies to new objects
for node_type, node_names in objects_apply_policy.items():
nodes = self.collection.find({
'name': {
'$in': node_names
},
'path':
get_filter_this_domain(domain),
'type':
node_type
})
for node in nodes:
apply_policies_function = globals()['apply_policies_to_%s'
% node['type']]
apply_policies_function(self.collection,
node,
admin_user,
api=chef_server_api)
# Return result
status = '{0} inserted, {1} updated of {2} objects imported successfully.'.format(
report['inserted'], report['updated'], report['total'])
response = {'status': status, 'ok': True}
except Exception as e:
logger.exception(e)
response = {'status': u'{0}'.format(e), 'ok': False}
warnings = report.get('warnings', [])
if warnings:
response['warnings'] = warnings
return response