def __call__(self, node, value):
from iscompatible import iscompatible, string_to_tuple
super(UpdateControlFileValidator, self).__call__(node,value)
request = pyramid.threadlocal.get_current_request()
controlfile = self.decompress + os.sep + 'control'
settings = get_current_registry().settings
api = get_chef_api(settings, request.user)
cookbook = get_cookbook(api, settings.get('chef.cookbook_name'))
if os.path.exists(controlfile):
gecoscc_require = cookbook_require = None
with open(controlfile,'r') as f:
for line in f:
if line.startswith('gecoscc'):
gecoscc_require = line
elif line.startswith('cookbook'):
cookbook_require = line
if gecoscc_require and not iscompatible(gecoscc_require, string_to_tuple(request.VERSION)):
node.raise_invalid(gettext(self.err_msg))
if cookbook_require and not iscompatible(cookbook_require, string_to_tuple(cookbook['version'])):
node.raise_invalid(gettext(self.err_msg))
def __call__(self, node, value):
from iscompatible import iscompatible, string_to_tuple
super(UpdateControlFileValidator, self).__call__(node,value)
request = pyramid.threadlocal.get_current_request()
controlfile = self.decompress + os.sep + 'control'
settings = get_current_registry().settings
api = get_chef_api(settings, request.user)
cookbook = get_cookbook(api, settings.get('chef.cookbook_name'))
if os.path.exists(controlfile):
gecoscc_require = cookbook_require = None
with open(controlfile,'r') as f:
for line in f:
if line.startswith('gecoscc'):
gecoscc_require = line
elif line.startswith('cookbook'):
cookbook_require = line
if gecoscc_require and not iscompatible(gecoscc_require, string_to_tuple(request.VERSION)):
node.raise_invalid(gettext(self.err_msg))
if cookbook_require and not iscompatible(cookbook_require, string_to_tuple(cookbook['version'])):
node.raise_invalid(gettext(self.err_msg))
def object_action(self, user, obj, objold=None, action=None, computers=None):
api = get_chef_api(self.app.conf, user)
cookbook = get_cookbook(api, self.app.conf.get('chef.cookbook_name'))
computers = computers or self.get_related_computers(obj)
are_new_jobs = False
for computer in computers:
try:
job_ids_by_computer = []
node_chef_id = computer.get('node_chef_id', None)
node = reserve_node_or_raise(node_chef_id, api, 'gcc-tasks-%s-%s' % (obj['_id'], random.random()), 10)
if not node.get(self.app.conf.get('chef.cookbook_name')):
raise NodeNotLinked("Node %s is not linked" % node_chef_id)
error_last_saved = computer.get('error_last_saved', False)
error_last_chef_client = computer.get('error_last_chef_client', False)
force_update = error_last_saved or error_last_chef_client
node, updated = self.update_node(user, computer, obj, objold, node, action, job_ids_by_computer, force_update)
if not updated:
save_node_and_free(node)
continue
are_new_jobs = True
self.validate_data(node, cookbook, api)
save_node_and_free(node)
if error_last_saved:
self.db.nodes.update({'_id': computer['_id']},
{'$set': {'error_last_saved': False}})
except NodeNotLinked as e:
self.report_node_not_linked(computer, user, obj, action)
are_new_jobs = True
save_node_and_free(node, api, refresh=True)
except NodeBusyException as e:
self.report_node_busy(computer, user, obj, action)
are_new_jobs = True
except ValidationError as e:
if not job_ids_by_computer:
self.report_unknown_error(e, user, obj, action, computer)
self.report_error(e, job_ids_by_computer, computer, 'Validation error: ')
save_node_and_free(node, api, refresh=True)
are_new_jobs = True
except Exception as e:
if not job_ids_by_computer:
self.report_unknown_error(e, user, obj, action, computer)
self.report_error(e, job_ids_by_computer, computer)
try:
save_node_and_free(node, api, refresh=True)
except:
pass
are_new_jobs = True
if are_new_jobs:
invalidate_jobs(self.request, user)
def command(self):
collection = self.db.serviceproviders
sp_model = ServiceProvider()
chef_ssl_verify = True if self.settings.get(
'chef.ssl.verify') == "True" else False
api = _get_chef_api(self.settings.get('chef.url'),
toChefUsername(self.options.chef_username),
self.options.chef_pem, chef_ssl_verify,
self.settings.get('chef.version'))
cookbook_name = self.settings['chef.cookbook_name']
cookbook = get_cookbook(api, cookbook_name)
for f in cookbook['files']:
if f['name'] == 'serviceproviders.xml':
try:
xml = requests.get(f['url'], verify=chef_ssl_verify)
break
except requests.exceptions.RequestException as e:
print e
sys.exit(1)
# Parsing XML
root = ET.fromstring(xml.text)
for country in root.findall('country'):
for providername in country.findall('provider/name'):
if providername.text:
try:
new_sp = sp_model.serialize({
'name':
country.get('code').lower(),
'provider':
providername.text
})
print new_sp
collection.insert(new_sp)
except:
print "ERROR:" + providername.text
def command(self):
collection = self.db.serviceproviders
sp_model = ServiceProvider()
chef_ssl_verify = True if self.settings.get('chef.ssl.verify') == "True" else False
api = _get_chef_api(self.settings.get('chef.url'),
toChefUsername(self.options.chef_username),
self.options.chef_pem, chef_ssl_verify, self.settings.get('chef.version'))
cookbook_name = self.settings['chef.cookbook_name']
cookbook = get_cookbook(api, cookbook_name)
for f in cookbook['files']:
if f['name'] == 'serviceproviders.xml':
try:
xml = requests.get(f['url'], verify=chef_ssl_verify)
break
except requests.exceptions.RequestException as e:
print e
sys.exit(1)
# Parsing XML
root = ET.fromstring(xml.text)
for country in root.findall('country'):
for providername in country.findall('provider/name'):
if providername.text:
try:
new_sp=sp_model.serialize({'name': country.get('code').lower(), 'provider': providername.text})
print new_sp
collection.insert(new_sp)
except:
print "ERROR:" + providername.text
def command(self):
api = _get_chef_api(self.settings.get('chef.url'),
toChefUsername(self.options.chef_username),
self.options.chef_pem)
cookbook_name = self.settings['chef.cookbook_name']
cookbook = get_cookbook(api, cookbook_name)
languages = self.settings.get('pyramid.locales')
languages.remove(self.settings.get('pyramid.default_locale_name'))
policies = {}
try:
for key, value in cookbook['metadata']['attributes']['json_schema']['object']['properties']['gecos_ws_mgmt']['properties'].items():
for k, policy in value['properties'].items():
policy['path'] = '%s.%s.%s' % (cookbook_name, key, k)
policies[k] = policy
except KeyError:
print "Can not found policies in cookbook %s" % cookbook_name
sys.exit(1)
policies_to_import = self.options.policies
if policies_to_import:
found = set(policies_to_import).intersection(set(policies.keys()))
not_found = set(policies_to_import).difference(set(policies.keys()))
if not_found:
print "%s policies to import. Policies NOT FOUND: %s" % (len(found), list(not_found))
else:
print "%s policies to import" % len(found)
else:
print "%s policies to import" % len(policies.keys())
for key, value in policies.items():
if policies_to_import and key not in policies_to_import:
continue
elif key in EXCLUDE_POLICIES:
continue
if key == PACKAGE_POLICY:
self.set_packages_url(value)
for ex_attr in EXCLUDE_GENERIC_ATTRS:
if ex_attr in value['properties']:
del(value['properties'][ex_attr])
path = value.pop('path')
support_os = value['properties']['support_os']['default']
del value['properties']['support_os']
if is_user_policy(path):
targets = ['ou', 'user', 'group']
title = value['title']
titles = {}
for lan in languages:
titles[lan] = value['title_' + lan]
value = value['properties']['users']['patternProperties']['.*']
if 'updated_by' in value.get('properties', {}):
del value['properties']['updated_by']
value['title'] = title
for lan in languages:
value['title_' + lan] = titles[lan]
elif 'network_mgmt' in path:
targets = ['computer']
else:
targets = DEFAULT_TARGETS
policy = {
'name': value['title'],
'slug': key,
'path': path,
'schema': value,
'targets': targets,
'is_emitter_policy': False,
'support_os': support_os,
}
for lan in languages:
policy['name_' + lan] = value['title_' + lan]
self.treatment_policy(policy)
self.create_software_profiles_policy(policies, languages)
if not self.options.ignore_emitter_policies:
for emiter in RESOURCES_EMITTERS_TYPES:
slug = emiter_police_slug(emiter)
schema = deepcopy(SCHEMA_EMITTER)
schema['properties']['object_related_list']['title'] = '%s list' % emiter.capitalize()
for lan in languages:
schema['properties']['object_related_list']['title_' + lan] = EMITTER_LIST_LOCALIZED[lan] % EMITTER_LOCALIZED[lan][emiter]
schema['properties']['object_related_list']['autocomplete_url'] = POLICY_EMITTER_URL[slug]
policy = {
'name': POLICY_EMITTER_NAMES[slug],
'slug': slug,
'path': POLICY_EMITTER_PATH[slug],
'targets': POLICY_EMITTER_TARGETS[slug],
'is_emitter_policy': True,
'schema': schema,
'support_os': policies[POLICY_EMITTER_PATH[slug].split('.')[2]]['properties']['support_os']['default']
}
for lan in languages:
policy['name_' + lan] = POLICY_EMITTER_NAMES_LOCALIZED[lan][slug]
self.treatment_policy(policy)
def command(self):
api = _get_chef_api(self.settings.get('chef.url'),
toChefUsername(self.options.chef_username),
self.options.chef_pem, self.settings.get('chef.ssl.verify'), self.settings.get('chef.version'))
cookbook_name = self.settings['chef.cookbook_name']
cookbook = get_cookbook(api, cookbook_name)
languages = self.settings.get('pyramid.locales')
languages.remove(self.settings.get('pyramid.default_locale_name'))
policies = {}
try:
for key, value in cookbook['metadata']['attributes']['json_schema']['object']['properties']['gecos_ws_mgmt']['properties'].items():
for k, policy in value['properties'].items():
policy['path'] = '%s.%s.%s' % (cookbook_name, key, k)
policies[k] = policy
except KeyError:
print "Can not found policies in cookbook %s" % cookbook_name
sys.exit(1)
policies_to_import = self.options.policies
if policies_to_import:
found = set(policies_to_import).intersection(set(policies.keys()))
not_found = set(policies_to_import).difference(set(policies.keys()))
if not_found:
print "%s policies to import. Policies NOT FOUND: %s" % (len(found), list(not_found))
else:
print "%s policies to import" % len(found)
else:
print "%s policies to import" % len(policies.keys())
for key, value in policies.items():
if policies_to_import and key not in policies_to_import:
continue
elif key in EXCLUDE_POLICIES:
continue
if key == PACKAGE_POLICY:
self.set_packages_url(value)
if key == MIMETYPES_POLICY:
self.set_mimetypes_url(value)
if key == MOBILE_BROADBAND_POLICY:
self.set_serviceproviders_url(value)
support_os = value['properties']['support_os']['default']
for ex_attr in EXCLUDE_GENERIC_ATTRS:
if ex_attr in value['properties']:
del(value['properties'][ex_attr])
path = value.pop('path')
form_layout = value.pop('form', {})
is_mergeable = value.pop('is_mergeable', False)
autoreverse = value.pop('autoreverse', False)
if is_user_policy(path):
targets = ['ou', 'user', 'group']
title = value['title']
titles = {}
for lan in languages:
titles[lan] = value['title_' + lan]
value = value['properties']['users']['patternProperties']['.*']
if 'updated_by' in value.get('properties', {}):
del value['properties']['updated_by']
value['title'] = title
for lan in languages:
value['title_' + lan] = titles[lan]
elif 'single_node' in path:
targets = ['computer']
else:
targets = DEFAULT_TARGETS
policy = {
'name': value['title'],
'slug': key,
'path': path,
'schema': value,
'form': form_layout,
'targets': targets,
'is_emitter_policy': False,
'support_os': support_os,
'is_mergeable': is_mergeable,
'autoreverse': autoreverse,
}
for lan in languages:
policy['name_' + lan] = value['title_' + lan]
self.treatment_policy(policy)
if not self.options.ignore_emitter_policies:
for emiter in RESOURCES_EMITTERS_TYPES:
slug = emiter_police_slug(emiter)
schema = deepcopy(SCHEMA_EMITTER)
schema['properties']['object_related_list']['title'] = '%s list' % emiter.capitalize()
for lan in languages:
schema['properties']['object_related_list']['title_' + lan] = EMITTER_LIST_LOCALIZED[lan] % EMITTER_LOCALIZED[lan][emiter]
schema['properties']['object_related_list']['autocomplete_url'] = POLICY_EMITTER_URL[slug]
policy = {
'name': POLICY_EMITTER_NAMES[slug],
'slug': slug,
'path': POLICY_EMITTER_PATH[slug],
'targets': POLICY_EMITTER_TARGETS[slug],
'is_emitter_policy': True,
'schema': schema,
'support_os': policies[POLICY_EMITTER_PATH[slug].split('.')[2]]['properties']['support_os']['default'],
'is_mergeable': True,
'autoreverse': policies[POLICY_EMITTER_PATH[slug].split('.')[-2]].get('autoreverse', False)
}
for lan in languages:
policy['name_' + lan] = POLICY_EMITTER_NAMES_LOCALIZED[lan][slug]
self.treatment_policy(policy)
# Check non imported policies
print "Check non imported policies..."
dbpolicies = self.db.policies.find({})
found = False
for policy in dbpolicies:
if (policy['slug'] not in policies and
not policy['slug'].endswith('_can_view')):
print "Policy '%s' wasn't imported. Probably is deprecated." % (policy['slug'])
found = True
if self.options.delete:
# Delete deprecated policy
self.db.policies.remove({'slug': policy['slug']})
print "Policy '%s' deleted!" % (policy['slug'])
if policy['slug'] == 'package_profile_res':
# Also delete software_profiles collection
print "Drop software profiles collection!"
self.db.software_profiles.drop()
self.db.settings.remove({'key' : 'software_profiles'})
if not found:
print "There are no deprecated policies"
def object_action(self,
user,
obj,
objold=None,
action=None,
computers=None):
api = get_chef_api(self.app.conf, user)
cookbook = get_cookbook(api, self.app.conf.get('chef.cookbook_name'))
computers = computers or self.get_related_computers(obj)
are_new_jobs = False
for computer in computers:
try:
job_ids_by_computer = []
node_chef_id = computer.get('node_chef_id', None)
node = reserve_node_or_raise(
node_chef_id, api,
'gcc-tasks-%s-%s' % (obj['_id'], random.random()), 10)
if not node.get(self.app.conf.get('chef.cookbook_name')):
raise NodeNotLinked("Node %s is not linked" % node_chef_id)
error_last_saved = computer.get('error_last_saved', False)
error_last_chef_client = computer.get('error_last_chef_client',
False)
force_update = error_last_saved or error_last_chef_client
node, updated = self.update_node(user, computer, obj, objold,
node, action,
job_ids_by_computer,
force_update)
if not updated:
save_node_and_free(node)
continue
are_new_jobs = True
self.validate_data(node, cookbook, api)
save_node_and_free(node)
if error_last_saved:
self.db.nodes.update({'_id': computer['_id']},
{'$set': {
'error_last_saved': False
}})
except NodeNotLinked as e:
self.report_node_not_linked(computer, user, obj, action)
are_new_jobs = True
save_node_and_free(node, api, refresh=True)
except NodeBusyException as e:
self.report_node_busy(computer, user, obj, action)
are_new_jobs = True
except ValidationError as e:
if not job_ids_by_computer:
self.report_unknown_error(e, user, obj, action, computer)
self.report_error(e, job_ids_by_computer, computer,
'Validation error: ')
save_node_and_free(node, api, refresh=True)
are_new_jobs = True
except Exception as e:
if not job_ids_by_computer:
self.report_unknown_error(e, user, obj, action, computer)
self.report_error(e, job_ids_by_computer, computer)
try:
save_node_and_free(node, api, refresh=True)
except:
pass
are_new_jobs = True
if are_new_jobs:
invalidate_jobs(self.request, user)
def command(self):
'''
This command recalculate the policies of the selected nodes
These nodes are receiving as command arguments
'''
db = self.pyramid.db
computers, num_computers = self.get_computers()
admin_user = db.adminusers.find_one(
{'username': self.options.administrator})
if not admin_user:
sys.stdout.write('Administrator does not exists\n')
sys.exit(1)
elif not admin_user.get('is_superuser', None):
sys.stdout.write('Administrator should be super user\n')
sys.exit(1)
api = get_chef_api(self.settings, admin_user)
cookbook_name = self.settings['chef.cookbook_name']
# It is not really the max dots, because the integer division.
max_optimal_dots = 80
total_dots = min(num_computers, max_optimal_dots)
if total_dots == num_computers:
step = 1
else:
step = old_div(num_computers, total_dots)
total_dots = int(math.ceil(float(num_computers) / step))
sys.stdout.write('%s 100%%\n' % ('.' * total_dots))
sys.stdout.flush()
results_error = {}
results_succes = {}
# Get the cookbook
cookbook = get_cookbook(api, cookbook_name)
schema = cookbook['metadata']['attributes']['json_schema']['object']
# Validate the cookbook schema
validator = validator_for(schema)
validator.check_schema(schema)
for i, comp in enumerate(list(computers)):
if i % step == 0:
sys.stdout.write('.')
sys.stdout.flush()
recalculated, reason = recalc_node_policies(
db.nodes, db.jobs, comp, admin_user, cookbook_name, api,
cookbook, validator)
if recalculated:
results_succes[comp['name']] = reason
else:
results_error[comp['name']] = reason
sys.stdout.write('\n\n\n*********** Success ********** \n')
for name, reason in list(results_succes.items()):
sys.stdout.write('%s: %s \n' % (name, reason))
sys.stdout.write('\n\n\n*********** Errors ********** \n')
for name, reason in list(results_error.items()):
sys.stdout.write('%s: %s \n' % (name, reason))
sys.stdout.flush()
