def get_technique_expected_reports(technique_metadata):
"""Generates technique expected reports from technique metadata"""
# Get all generic methods
generic_methods = ncf.get_all_generic_methods_metadata()
# Content start with a header
content = [
"""# This file contains one line per report expected by Rudder from this technique
# Format: technique_name;;class_prefix_${key};;@@RUDDER_ID@@;;component name;;component key"""
]
technique_name = technique_metadata['bundle_name']
for method_call in technique_metadata["method_calls"]:
method_name = method_call['method_name']
generic_method = generic_methods[method_name]
component = generic_method['name']
key_value = method_call["args"][generic_method["class_parameter_id"] -
1]
class_prefix = generic_method["class_prefix"] + "_" + key_value
line = technique_name + ";;" + class_prefix + ";;@@RUDDER_ID@@;;" + component + ";;" + key_value
content.append(line)
# Join all lines + last line
result = "\n".join(content) + "\n"
return result
def get_technique_metadata_xml(technique_metadata):
"""Get metadata xml for a technique as string"""
# Get all generic methods
generic_methods = ncf.get_all_generic_methods_metadata()
content = []
content.append('<TECHNIQUE name="'+technique_metadata['name']+'">')
content.append(' <DESCRIPTION>'+technique_metadata['description']+'</DESCRIPTION>')
content.append(' <BUNDLES>')
content.append(' <NAME>'+ technique_metadata['bundle_name'] + '</NAME>')
content.append(' </BUNDLES>')
content.append(' <SECTIONS>')
method_calls = technique_metadata["method_calls"]
# Get all method call, with no duplicate values
methods_name = set()
for method_call in method_calls:
method_name = methods_name.add(method_call['method_name'])
# For each method used, create a section containing all calls to that method
section_list = []
for method_name in methods_name:
try:
generic_method = generic_methods[method_name]
except Exception, e:
print "Error: The method '" + method_name + "' does not exist. Aborting Technique creation..."
# Filter all method calls to get only those about that method
filter_method_calls = [x for x in method_calls if x["method_name"] == method_name]
# Generare xml for that section
section = generate_section_xml(filter_method_calls, generic_method)
section_list.extend(section)
def get_technique_expected_reports(technique_metadata):
"""Generates technique expected reports from technique metadata"""
# Get all generic methods
generic_methods = ncf.get_all_generic_methods_metadata(
alt_path='/var/rudder/configuration-repository/ncf'
)["data"]['generic_methods']
# Content start with a header
content = [
"""# This file contains one line per report expected by Rudder from this technique
# Format: technique_name;;class_prefix_${key};;@@RUDDER_ID@@;;component name;;component key"""
]
technique_name = technique_metadata['bundle_name']
for method_call in technique_metadata["method_calls"]:
# Expected reports for Rudder should not include any "meta" bundle calls (any beginning with _)
if method_call['method_name'].startswith("_"):
continue
method_name = method_call['method_name']
generic_method = generic_methods[method_name]
component = generic_method['name']
key_value = method_call["args"][generic_method["class_parameter_id"] -
1]
class_prefix = (generic_method["class_prefix"] + "_" +
key_value).replace("\\'", "\'").replace('\\"', '\"')
line = technique_name + ";;" + class_prefix + ";;@@RUDDER_ID@@;;" + component + ";;" + key_value
content.append(line)
# Join all lines + last line
result = "\n".join(content) + "\n"
return result
def get_technique_expected_reports(technique_metadata):
"""Generates technique expected reports from technique metadata"""
# Get all generic methods
generic_methods = ncf.get_all_generic_methods_metadata(alt_path='/var/rudder/configuration-repository/ncf')["data"]['generic_methods']
# Content start with a header
content = ["""# This file contains one line per report expected by Rudder from this technique
# Format: technique_name;;class_prefix_${key};;@@RUDDER_ID@@;;component name;;component key"""]
technique_name = technique_metadata['bundle_name']
for method_call in technique_metadata["method_calls"]:
# Expected reports for Rudder should not include any "meta" bundle calls (any beginning with _)
if method_call['method_name'].startswith("_"):
continue
method_name = method_call['method_name']
generic_method = generic_methods[method_name]
component = generic_method['name']
key_value = method_call["args"][generic_method["class_parameter_id"]-1]
class_prefix = (generic_method["class_prefix"]+"_"+key_value).replace("\\'", "\'").replace('\\"', '\"')
line = technique_name+";;"+class_prefix+";;@@RUDDER_ID@@;;"+component+";;"+key_value
content.append(line)
# Join all lines + last line
result = "\n".join(content)+"\n"
return result
def test_get_all_generic_methods_metadata_with_arg(self):
"""get_all_generic_methods_metadata should return a list of all generic_methods with all defined metadata tags"""
metadata = ncf.get_all_generic_methods_metadata(
)["data"]["generic_methods"]
number_generic_methods = len(ncf.get_all_generic_methods_filenames())
self.assertEqual(number_generic_methods, len(metadata))
def get_technique_expected_reports(technique_metadata):
"""Generates technique expected reports from technique metadata"""
# Get all generic methods
generic_methods = ncf.get_all_generic_methods_metadata()
# Content start with a header
content = [
"""# This file contains one line per report expected by Rudder from this technique
# Format: technique_name;;class_prefix_${key};;@@RUDDER_ID@@;;component name;;component key"""
]
technique_name = technique_metadata["bundle_name"]
for method_call in technique_metadata["method_calls"]:
# Expected reports for Rudder should not include any "meta" bundle calls (any beginning with _)
if method_call["method_name"].startswith("_"):
continue
method_name = method_call["method_name"]
generic_method = generic_methods[method_name]
component = generic_method["name"]
key_value = method_call["args"][generic_method["class_parameter_id"] - 1]
class_prefix = generic_method["class_prefix"] + "_" + key_value
line = technique_name + ";;" + class_prefix + ";;@@RUDDER_ID@@;;" + component + ";;" + key_value
content.append(line)
# Join all lines + last line
result = "\n".join(content) + "\n"
return result
def setUp(self):
self.test_technique_file = os.path.realpath('test_technique.cf')
self.test_generic_method_file = 'test_generic_method.cf'
with open(self.test_technique_file) as fd:
self.technique_content = fd.read()
with open(self.test_generic_method_file) as fd:
self.generic_method_content = fd.read()
self.all_methods = ncf.get_all_generic_methods_metadata()["data"]["generic_methods"]
self.technique_metadata = ncf.parse_technique_metadata(self.technique_content)['result']
method_calls = ncf.parse_technique_methods(self.test_technique_file)
self.technique_metadata['method_calls'] = method_calls
self.technique_metadata_test = { 'name': 'ncf technique method argument escape test', 'description': "This is a bundle to test ncf's Python lib", 'version': '0.1', 'bundle_name': 'content_escaping_test', 'bundle_args': [],
'method_calls': [
{ 'method_name': 'package_install_version', 'args': ['apache2', '2.2.11'], 'class_context': 'any' },
{ 'method_name': 'file_replace_lines', 'args': ['/etc/httpd/conf/httpd.conf', 'ErrorLog \"/var/log/httpd/error_log\"', 'ErrorLog "/projet/logs/httpd/error_log"'], 'class_context': 'redhat' },
]
}
self.technique_metadata_test_content = os.path.realpath('technique_metadata_test_content.cf')
all_tags = ncf.tags["generic_method"]+ncf.tags["common"]
self.methods_expected_tags = [ tag for tag in all_tags if not tag in ncf.optionnal_tags ]
with open(self.technique_metadata_test_content) as fd:
self.technique_test_expected_content = fd.read()
def test_get_all_generic_methods_metadata_with_arg(self):
"""get_all_generic_methods_metadata should return a list of all generic_methods with all defined metadata tags"""
alternative_path = os.path.dirname(os.path.realpath(__file__)) + "/test_methods"
metadata = ncf.get_all_generic_methods_metadata(alternative_path)["data"]
number_generic_methods = len(ncf.get_all_generic_methods_filenames(alternative_path))
self.assertEquals(number_generic_methods, len(metadata))
def setUp(self):
self.test_technique_file = os.path.realpath('test_technique.cf')
self.test_generic_method_file = 'test_generic_method.cf'
with open(self.test_technique_file) as fd:
self.technique_content = fd.read()
with open(self.test_generic_method_file) as fd:
self.generic_method_content = fd.read()
self.all_methods = ncf.get_all_generic_methods_metadata(
)["data"]["generic_methods"]
self.technique_metadata = ncf.parse_technique_metadata(
self.technique_content)['result']
method_calls = ncf.parse_technique_methods(self.test_technique_file,
self.all_methods)
self.technique_metadata['method_calls'] = method_calls
self.technique_metadata_test = {
'name':
'ncf technique method argument escape test',
'description':
"This is a bundle to test ncf's Python lib",
'version':
'0.1',
'bundle_name':
'content_escaping_test',
'bundle_args': [],
'parameter': [],
'method_calls': [
{
'method_name': 'package_install_version',
"component": "Install a package with correct version",
'args': ['apache2', '2.2.11'],
'class_context': 'any'
},
{
'method_name':
'file_replace_lines',
"component":
"Edit conf file",
'args': [
'/etc/httpd/conf/httpd.conf',
'ErrorLog \"/var/log/httpd/error_log\"',
'ErrorLog "/projet/logs/httpd/error_log"'
],
'class_context':
'redhat'
},
]
}
self.technique_metadata_test_content = os.path.realpath(
'technique_metadata_test_content.cf')
all_tags = ncf.tags["generic_method"]
self.methods_expected_tags = [
tag for tag in all_tags
if not tag in ncf.optionnal_tags["generic_method"]
]
with open(self.technique_metadata_test_content) as fd:
self.technique_test_expected_content = fd.read().split("\n")
def generate_rudder_reporting(technique):
"""Generate complementary reporting needed for Rudder in rudder_reporting.st file"""
# Get all generic methods
generic_methods = ncf.get_all_generic_methods_metadata(alt_path='/var/rudder/configuration-repository/ncf')['data']['generic_methods']
content = []
content.append('bundle agent '+ technique['bundle_name']+'_rudder_reporting')
content.append('{')
content.append(' vars:')
content.append(' "promisers" slist => { @{this.callers_promisers}, cf_null }, policy => "ifdefined";')
content.append(' "class_prefix" string => canonify(join("_", "promisers"));')
content.append(' "args" slist => { };')
content.append('')
content.append(' methods:')
# Handle method calls
technique_name = technique['bundle_name']
# Filter calls so we only have those who are not using a any class_context
filter_calls = [ method_call for method_call in technique["method_calls"] if method_call['class_context'] != "any" ]
for method_call in filter_calls:
method_name = method_call['method_name']
generic_method = generic_methods[method_name]
key_value = method_call["args"][generic_method["class_parameter_id"]-1].replace("\\'", "\'")
regex = re.compile("[^\$\{\}a-zA-Z0-9_](?![^{}]+})|\$(?!{)")
# to match cfengine behaviour we need to treat utf8 as if it was ascii (see #7195)
# string should be unicode string (ie u'') which is the case if they are read from files opened with encoding="utf-8"
key_value = key_value.encode("utf-8").decode("iso-8859-1")
key_value_canonified = regex.sub("_", key_value)
class_prefix = generic_method["class_prefix"]+"_"+key_value_canonified
logger_rudder_call = '"dummy_report" usebundle => log_rudder("' + generic_method['name'] + ' ' + key_value + ' if ' + method_call['class_context'] + '", "' + class_prefix +'", "${class_prefix}", @{args})'
logger_rudder_call = logger_rudder_call.replace("&", "\\&")
# Always add an empty line for readability
content.append('')
if not "$" in method_call['class_context']:
content.append(' !('+method_call['class_context']+')::')
content.append(' "dummy_report" usebundle => _classes_noop("'+class_prefix+'");')
content.append(' ' + logger_rudder_call + ';')
else:
class_context = ncf.canonify_class_context(method_call['class_context'])
content.append(' "dummy_report" usebundle => _classes_noop("'+class_prefix+'"),')
content.append(' ifvarclass => concat("!('+class_context+')");')
content.append(' ' + logger_rudder_call + ',')
content.append(' ifvarclass => concat("!('+class_context+')");')
content.append('}')
# Join all lines with \n to get a pretty CFEngine file
result = '\n'.join(content)+"\n"
return result
def get_technique_metadata_xml(technique_metadata, include_rudder_reporting = False):
"""Get metadata xml for a technique as string"""
# Get all generic methods
generic_methods = ncf.get_all_generic_methods_metadata(alt_path='/var/rudder/configuration-repository/ncf')['data']
content = []
content.append('<TECHNIQUE name="'+technique_metadata['name']+'">')
content.append(' <DESCRIPTION>'+technique_metadata['description']+'</DESCRIPTION>')
content.append(' <BUNDLES>')
content.append(' <NAME>'+ technique_metadata['bundle_name'] + '</NAME>')
if include_rudder_reporting:
content.append(' <NAME>'+ technique_metadata['bundle_name'] + '_rudder_reporting</NAME>')
content.append(' </BUNDLES>')
if include_rudder_reporting:
content.append(' <TMLS>')
content.append(' <TML name="rudder_reporting"/>')
content.append(' </TMLS>')
content.append(' <SECTIONS>')
method_calls = technique_metadata["method_calls"]
# Get all method call, with no duplicate values
methods_name = set()
for method_call in method_calls:
# Expected reports for Rudder should not include any "meta" bundle calls (any beginning with _)
if method_call['method_name'].startswith("_"):
continue
method_name = methods_name.add(method_call['method_name'])
# For each method used, create a section containing all calls to that method
section_list = []
for method_name in methods_name:
try:
generic_method = generic_methods[method_name]
except Exception as e:
sys.stderr.write("Error: The method '" + method_name + "' does not exist. Aborting Technique creation..." + "\n")
sys.stderr.write(traceback.format_exc())
exit(1)
# Filter all method calls to get only those about that method
filter_method_calls = [x for x in method_calls if x["method_name"] == method_name]
# Generare xml for that section
section = generate_section_xml(filter_method_calls, generic_method)
section_list.extend(section)
content.extend(section_list)
content.append(' </SECTIONS>')
content.append('</TECHNIQUE>')
# Join all lines with \n to get a pretty xml
result = '\n'.join(content)+"\n"
return result
def test_get_all_generic_methods_metadata_with_arg(self):
"""get_all_generic_methods_metadata should return a list of all generic_methods with all defined metadata tags"""
alternative_path = os.path.dirname(
os.path.realpath(__file__)) + "/test_methods"
metadata = ncf.get_all_generic_methods_metadata(
alternative_path)["data"]
number_generic_methods = len(
ncf.get_all_generic_methods_filenames(alternative_path))
self.assertEqual(number_generic_methods, len(metadata))
def get_generic_methods():
"""Get all generic methods from ncf folder passed as parameter, or default ncf folder if not defined"""
# We need to get path from url params, if not present put "" as default value
if "path" in request.args:
path = request.args['path']
else:
path = ""
generic_methods = ncf.get_all_generic_methods_metadata(alt_path = path)
resp = jsonify( generic_methods )
return resp
def get_generic_methods():
"""Get all generic methods from ncf folder passed as parameter, or default ncf folder if not defined"""
try:
# We need to get path from url params, if not present put "" as default value
path = get_path_from_args(request)
generic_methods = ncf.get_all_generic_methods_metadata(alt_path = path)
resp = jsonify( generic_methods )
return resp
except Exception as e:
return format_error(e, "generic methods fetching", 500)
def get_generic_methods():
"""Get all generic methods from ncf folder passed as parameter, or default ncf folder if not defined"""
try:
# We need to get path from url params, if not present put "" as default value
path = get_path_from_args(request)
generic_methods = ncf.get_all_generic_methods_metadata(alt_path=path)
resp = jsonify(generic_methods)
return resp
except Exception as e:
return format_error(e, "generic methods fetching", 500)
def write_technique_for_rudder(root_path, technique):
""" From a technique, generate all files needed for Rudder in specified path"""
path = get_path_for_technique(root_path,technique)
if not os.path.exists(path):
os.makedirs(path)
# We don't need to create rudder_reporting.st if all class context are any
generic_methods = ncf.get_all_generic_methods_metadata(alt_path='/var/rudder/configuration-repository/ncf')['data']['generic_methods']
include_rudder_reporting = not all(method_call['class_context'] == 'any' and
"cfengine-community" in generic_methods[method_call['method_name']]["agent_support"] for method_call in technique["method_calls"])
if include_rudder_reporting:
write_rudder_reporting_file(path,technique)
def write_technique_for_rudder(root_path, technique):
""" From a technique, generate all files needed for Rudder in specified path"""
path = get_path_for_technique(root_path,technique)
if not os.path.exists(path):
os.makedirs(path)
# We don't need to create rudder_reporting.st if all class context are any
generic_methods = ncf.get_all_generic_methods_metadata(alt_path='/var/rudder/configuration-repository/ncf')['data']['generic_methods']
include_rudder_reporting = not all(method_call['class_context'] == 'any' and
"cfengine-community" in generic_methods[method_call['method_name']]["agent_support"] for method_call in technique["method_calls"])
write_expected_reports_file(path,technique)
if include_rudder_reporting:
write_rudder_reporting_file(path,technique)
def generate_rudder_reporting(technique):
"""Generate complementary reporting needed for Rudder in rudder_reporting.st file"""
# Get all generic methods
generic_methods = ncf.get_all_generic_methods_metadata(alt_path='/var/rudder/configuration-repository/ncf')
content = []
content.append('bundle agent '+ technique['bundle_name']+'_rudder_reporting')
content.append('{')
content.append(' methods:')
# Handle method calls
technique_name = technique['bundle_name']
# Filter calls so we only have those who are not using a any class_context
filter_calls = [ method_call for method_call in technique["method_calls"] if method_call['class_context'] != "any" ]
for method_call in filter_calls:
method_name = method_call['method_name']
generic_method = generic_methods[method_name]
key_value = method_call["args"][generic_method["class_parameter_id"]-1]
key_value_canonified = re.sub("[^\$\{\}\w](?![^{}]+})|\$(?!{)", "_", key_value)
class_prefix = generic_method["class_prefix"]+"_"+key_value_canonified
# Always add an empty line for readability
content.append('')
if not "$" in method_call['class_context']:
content.append(' !('+method_call['class_context']+')::')
content.append(' "dummy_report" usebundle => _classes_noop("'+class_prefix+'");')
content.append(' "dummy_report" usebundle => logger_rudder("Not applicable", "'+class_prefix+'");')
else:
content.append(' "dummy_report" usebundle => _classes_noop("'+class_prefix+'"),')
content.append(' ifvarclass => "!('+method_call['class_context']+')";')
content.append(' "dummy_report" usebundle => logger_rudder("Not applicable", "'+class_prefix+'"),')
content.append(' ifvarclass => "!('+method_call['class_context']+')";')
content.append('}')
# Join all lines with \n to get a pretty CFEngine file
result = '\n'.join(content)+"\n"
return result
def get_technique_metadata_xml(technique_metadata, include_rudder_reporting=False):
"""Get metadata xml for a technique as string"""
# Get all generic methods
generic_methods = ncf.get_all_generic_methods_metadata(alt_path="/var/rudder/configuration-repository/ncf")
content = []
content.append('<TECHNIQUE name="' + technique_metadata["name"] + '">')
content.append(" <DESCRIPTION>" + technique_metadata["description"] + "</DESCRIPTION>")
content.append(" <BUNDLES>")
content.append(" <NAME>" + technique_metadata["bundle_name"] + "</NAME>")
if include_rudder_reporting:
content.append(" <NAME>" + technique_metadata["bundle_name"] + "_rudder_reporting</NAME>")
content.append(" </BUNDLES>")
if include_rudder_reporting:
content.append(" <TMLS>")
content.append(' <TML name="rudder_reporting"/>')
content.append(" </TMLS>")
content.append(" <SECTIONS>")
method_calls = technique_metadata["method_calls"]
# Get all method call, with no duplicate values
methods_name = set()
for method_call in method_calls:
# Expected reports for Rudder should not include any "meta" bundle calls (any beginning with _)
if method_call["method_name"].startswith("_"):
continue
method_name = methods_name.add(method_call["method_name"])
# For each method used, create a section containing all calls to that method
section_list = []
for method_name in methods_name:
try:
generic_method = generic_methods[method_name]
except Exception, e:
print "Error: The method '" + method_name + "' does not exist. Aborting Technique creation..."
# Filter all method calls to get only those about that method
filter_method_calls = [x for x in method_calls if x["method_name"] == method_name]
# Generare xml for that section
section = generate_section_xml(filter_method_calls, generic_method)
section_list.extend(section)
def convert_all_to_dsc(root_path):
techniques = ncf.get_all_techniques_metadata(
alt_path=rudder_ncf_path)['data']['techniques']
methods = ncf.get_all_generic_methods_metadata(
alt_path=rudder_ncf_path)['data']['generic_methods']
check_dsc_techniques_path()
ret = 0
for technique, metadata in techniques.items():
try:
convert_technique(metadata, methods)
except Exception as e:
sys.stderr.write(
"Error: Unable to create Rudder Technique files related to ncf Technique "
+ technique + ", skipping... (" + str(e) + ")\n")
sys.stderr.write(traceback.format_exc())
ret = 1
continue
exit(ret)
def setUp(self):
self.maxDiff = None
self.test_technique_file = os.path.realpath('test_technique.cf')
with open(self.test_technique_file) as fd:
self.technique_content = fd.read()
self.all_methods = ncf.get_all_generic_methods_metadata()["data"]["generic_methods"]
self.technique_metadata = ncf.parse_technique_metadata(self.technique_content)['result']
method_calls = ncf.parse_technique_methods(self.test_technique_file, self.all_methods)
self.technique_metadata['method_calls'] = method_calls
self.test_reporting = os.path.realpath('test_technique_reporting.cf')
with open(self.test_reporting) as fd:
self.reporting_content = fd.read()
self.reporting_metadata = ncf.parse_technique_metadata(self.reporting_content)['result']
reporting_method_calls = ncf.parse_technique_methods(self.test_reporting, self.all_methods)
self.reporting_metadata['method_calls'] = reporting_method_calls
self.test_reporting_with_var = os.path.realpath('test_technique_with_variable.cf')
with open(self.test_reporting_with_var) as fd:
self.reporting_with_var_content = fd.read()
self.reporting_with_var_metadata = ncf.parse_technique_metadata(self.reporting_with_var_content)['result']
reporting_with_var_method_calls = ncf.parse_technique_methods(self.test_reporting_with_var, self.all_methods)
self.reporting_with_var_metadata['method_calls'] = reporting_with_var_method_calls
self.test_any_technique = os.path.realpath('test_technique_any.cf')
with open(self.test_any_technique) as fd:
self.any_technique_content = fd.read()
self.any_technique_metadata = ncf.parse_technique_metadata(self.any_technique_content)['result']
any_technique_method_calls = ncf.parse_technique_methods(self.test_any_technique, self.all_methods)
self.any_technique_metadata['method_calls'] = any_technique_method_calls
self.rudder_reporting_file = os.path.realpath('test_technique_rudder_reporting.cf')
with open(self.rudder_reporting_file) as fd:
self.rudder_reporting_content = fd.read()
# Testing Techniques with quote
self.test_technique_with_quote_file = os.path.realpath('test_technique_with_quote.cf')
with open(self.test_technique_with_quote_file) as fd:
self.technique_with_quote_content = fd.read()
self.technique_with_quote_metadata = ncf.parse_technique_metadata(self.technique_with_quote_content)['result']
method_with_quote_calls = ncf.parse_technique_methods(self.test_technique_with_quote_file, self.all_methods)
self.technique_with_quote_metadata['method_calls'] = method_with_quote_calls
def setUp(self):
self.technique_file = os.path.realpath(
'technique_metadata_test_content.cf')
with open(self.technique_file) as fd:
self.technique_content = fd.read()
self.all_methods = ncf.get_all_generic_methods_metadata(
)["data"]["generic_methods"]
# For test, for now add dsc_support to all method
for method in self.all_methods:
self.all_methods[method]["agent_support"] = ["dsc"]
self.technique_metadata = ncf.parse_technique_metadata(
self.technique_content)['result']
method_calls = ncf.parse_technique_methods(self.technique_file)
self.technique_metadata['method_calls'] = method_calls
self.dsc_technique_file = os.path.realpath('test_technique.ps1')
with open(self.dsc_technique_file) as fd:
self.dsc_content = fd.read()
def get_technique_metadata_xml(technique_metadata):
"""Get metadata xml for a technique as string"""
# Get all generic methods
generic_methods = ncf.get_all_generic_methods_metadata()
content = []
content.append('<TECHNIQUE name="' + technique_metadata['name'] + '">')
content.append(' <DESCRIPTION>' + technique_metadata['description'] +
'</DESCRIPTION>')
content.append(' <BUNDLES>')
content.append(' <NAME>' + technique_metadata['bundle_name'] +
'</NAME>')
content.append(' </BUNDLES>')
content.append(' <SECTIONS>')
method_calls = technique_metadata["method_calls"]
# Get all method call, with no duplicate values
methods_name = set()
for method_call in method_calls:
method_name = methods_name.add(method_call['method_name'])
# For each method used, create a section containing all calls to that method
section_list = []
for method_name in methods_name:
try:
generic_method = generic_methods[method_name]
except Exception, e:
print "Error: The method '" + method_name + "' does not exist. Aborting Technique creation..."
# Filter all method calls to get only those about that method
filter_method_calls = [
x for x in method_calls if x["method_name"] == method_name
]
# Generare xml for that section
section = generate_section_xml(filter_method_calls, generic_method)
section_list.extend(section)
def convert_one_to_dsc(destination_path, bundle_name):
techniques = ncf.get_all_techniques_metadata(
alt_path=rudder_ncf_path)['data']['techniques']
methods = ncf.get_all_generic_methods_metadata(
alt_path=rudder_ncf_path)['data']['generic_methods']
check_dsc_techniques_path()
if bundle_name in techniques.keys():
try:
metadata = techniques[bundle_name]
convert_technique(metadata, methods)
except Exception as e:
sys.stderr.write(
"Error: Unable to create Rudder Technique files related to ncf Technique "
+ bundle_name + " (" + str(e) + ")\n")
sys.stderr.write(traceback.format_exc())
exit(1)
else:
sys.stderr.write(
"Error: Unable to create Rudder Technique files related to ncf Technique "
+ bundle_name + ", cannot find ncf Technique " + bundle_name +
"\n")
sys.stderr.write(traceback.format_exc())
exit(1)
def generate_rudder_reporting(technique):
"""Generate complementary reporting needed for Rudder in rudder_reporting.st file"""
# Get all generic methods
generic_methods = ncf.get_all_generic_methods_metadata()
content = []
content.append("bundle agent " + technique["bundle_name"] + "_rudder_reporting")
content.append("{")
content.append(" methods:")
# Handle method calls
technique_name = technique["bundle_name"]
# Filter calls so we only have those who are not using a any class_context
filter_calls = [method_call for method_call in technique["method_calls"] if method_call["class_context"] != "any"]
for method_call in filter_calls:
content.append(" !(" + method_call["class_context"] + ")::")
method_name = method_call["method_name"]
generic_method = generic_methods[method_name]
key_value = method_call["args"][generic_method["class_parameter_id"] - 1]
key_value_canonified = re.sub("[^a-zA-Z0-9_]", "_", key_value)
class_prefix = generic_method["class_prefix"] + "_" + key_value_canonified
content.append(' "dummy_report" usebundle => _classes_noop("' + class_prefix + '");')
content.append(' "dummy_report" usebundle => logger_rudder("Not applicable", "' + class_prefix + '");')
content.append("}")
# Join all lines with \n to get a pretty CFEngine file
result = "\n".join(content) + "\n"
return result
def test_get_all_generic_methods_metadata(self):
"""get_all_generic_methods_metadata should return a list of all generic_methods with all defined metadata tags"""
metadata = ncf.get_all_generic_methods_metadata()["data"]
number_generic_methods = len(ncf.get_all_generic_methods_filenames())
self.assertEquals(number_generic_methods, len(metadata))
def get_technique_metadata_xml(technique_metadata,
include_rudder_reporting=False):
"""Get metadata xml for a technique as string"""
# Get all generic methods
generic_methods = ncf.get_all_generic_methods_metadata(
alt_path='/var/rudder/configuration-repository/ncf'
)['data']['generic_methods']
content = []
content.append('<TECHNIQUE name="' + technique_metadata['name'] + '">')
content.append(' <DESCRIPTION>' + technique_metadata['description'] +
'</DESCRIPTION>')
content.append(' <BUNDLES>')
content.append(' <NAME>' + technique_metadata['bundle_name'] +
'</NAME>')
if include_rudder_reporting:
content.append(' <NAME>' + technique_metadata['bundle_name'] +
'_rudder_reporting</NAME>')
content.append(' </BUNDLES>')
if include_rudder_reporting:
content.append(' <TMLS>')
content.append(' <TML name="rudder_reporting"/>')
content.append(' </TMLS>')
content.append(' <FILES>')
content.append(
' <FILE name="RUDDER_CONFIGURATION_REPOSITORY/ncf/50_techniques/' +
technique_metadata['bundle_name'] + '/' +
technique_metadata['bundle_name'] + '.cf">')
content.append(' <INCLUDED>true</INCLUDED>')
content.append(' </FILE>')
content.append(' </FILES>')
content.append(' <SECTIONS>')
method_calls = technique_metadata["method_calls"]
# Get all method call, with no duplicate values
methods_name = set()
for method_call in method_calls:
# Expected reports for Rudder should not include any "meta" bundle calls (any beginning with _)
if method_call['method_name'].startswith("_"):
continue
method_name = methods_name.add(method_call['method_name'])
# For each method used, create a section containing all calls to that method
methods_name_ordered = list(methods_name)
methods_name_ordered.sort()
section_list = []
for method_name in methods_name_ordered:
try:
generic_method = generic_methods[method_name]
except Exception as e:
sys.stderr.write(
"Error: The method '" + method_name +
"' does not exist. Aborting Technique creation..." + "\n")
sys.stderr.write(traceback.format_exc())
exit(1)
# Filter all method calls to get only those about that method
filter_method_calls = [
x for x in method_calls if x["method_name"] == method_name
]
# Generare xml for that section
section = generate_section_xml(filter_method_calls, generic_method)
section_list.extend(section)
content.extend(section_list)
content.append(' </SECTIONS>')
content.append('</TECHNIQUE>')
# Join all lines with \n to get a pretty xml
result = '\n'.join(content) + "\n"
return result
def generate_rudder_reporting(technique):
"""Generate complementary reporting needed for Rudder in rudder_reporting.st file"""
# Get all generic methods
generic_methods = ncf.get_all_generic_methods_metadata(
alt_path='/var/rudder/configuration-repository/ncf'
)['data']['generic_methods']
content = []
content.append('bundle agent ' + technique['bundle_name'] +
'_rudder_reporting')
content.append('{')
content.append(' vars:')
content.append(
' "promisers" slist => { @{this.callers_promisers}, cf_null }, policy => "ifdefined";'
)
content.append(
' "class_prefix" string => canonify(join("_", "promisers"));')
content.append(' "args" slist => { };')
content.append('')
content.append(' methods:')
# Handle method calls
technique_name = technique['bundle_name']
# Filter calls so we only have those who are not using a any class_context
filter_calls = [
method_call for method_call in technique["method_calls"]
if method_call['class_context'] != "any"
]
for method_call in filter_calls:
method_name = method_call['method_name']
generic_method = generic_methods[method_name]
key_value = method_call["args"][generic_method["class_parameter_id"] -
1].replace("\\'", "\'")
regex = re.compile("[^\$\{\}a-zA-Z0-9_](?![^{}]+})|\$(?!{)")
# to match cfengine behaviour we need to treat utf8 as if it was ascii (see #7195)
# string should be unicode string (ie u'') which is the case if they are read from files opened with encoding="utf-8"
key_value = key_value.encode("utf-8").decode("iso-8859-1")
key_value_canonified = regex.sub("_", key_value)
class_prefix = generic_method[
"class_prefix"] + "_" + key_value_canonified
logger_rudder_call = '"dummy_report" usebundle => log_rudder("' + generic_method[
'name'] + ' ' + key_value + ' if ' + method_call[
'class_context'] + '", "' + class_prefix + '", "${class_prefix}", @{args})'
logger_rudder_call = logger_rudder_call.replace("&", "\\&")
# Always add an empty line for readability
content.append('')
if not "$" in method_call['class_context']:
content.append(' !(' + method_call['class_context'] + ')::')
content.append(
' "dummy_report" usebundle => _classes_noop("' +
class_prefix + '");')
content.append(' ' + logger_rudder_call + ';')
else:
class_context = ncf.canonify_class_context(
method_call['class_context'])
content.append(
' "dummy_report" usebundle => _classes_noop("' +
class_prefix + '"),')
content.append(' ifvarclass => concat("!(' +
class_context + ')");')
content.append(' ' + logger_rudder_call + ',')
content.append(' ifvarclass => concat("!(' +
class_context + ')");')
content.append('}')
# Join all lines with \n to get a pretty CFEngine file
result = '\n'.join(content) + "\n"
return result
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Test if the documentation fields contain unescaped dollar characters that would break pdflatex
import ncf
import re
import sys
from pprint import pprint
if __name__ == '__main__':
# Get all generic methods
generic_methods = ncf.get_all_generic_methods_metadata()["data"]["generic_methods"]
test_file = sys.argv[0]
check_backquotes = re.compile('[^\`]*\$[^\`]*')
errors = 0
for name, method in generic_methods.items():
if "documentation" in method:
if check_backquotes.match(method["documentation"]):
print("Test "+name+" has non escaped $ in its documentation")
errors += 1
if errors == 0:
print("R: "+test_file+" Pass")
else:
print("R: "+test_file+" FAIL")
def generate_rudder_reporting(technique):
"""Generate complementary reporting needed for Rudder in rudder_reporting.st file"""
# Get all generic methods
generic_methods = ncf.get_all_generic_methods_metadata(alt_path='/var/rudder/configuration-repository/ncf')['data']['generic_methods']
args = ""
bundle_param = ""
if len(technique["parameter"]) > 0:
bundle_param = "("+", ".join([ncf.canonify(param["name"]) for param in technique["parameter"] ])+")"
args = ", ".join('"${' + ncf.canonify(param["name"]) + '}"' for param in technique["parameter"] )
bundle_name = technique['bundle_name']+'_rudder_reporting'
content = []
content.append('bundle agent '+ technique['bundle_name']+'_rudder_reporting'+ bundle_param)
content.append('{')
content.append(' vars:')
content.append(' "args" slist => { '+ args +' };')
content.append(' "report_param" string => join("_", args);')
content.append(' "full_class_prefix" string => canonify("' + bundle_name + '_${report_param}");')
content.append(' "class_prefix" string => string_head("${full_class_prefix}", "1000");')
content.append('')
content.append(' methods:')
report_unique_id = 0
for method_call in technique["method_calls"]:
method_name = method_call['method_name']
if method_call['method_name'].startswith("_"):
continue
generic_method = generic_methods[method_name]
key_value = ncf.get_key_value(method_call, generic_method)
class_prefix = '${class_prefix}_'+ncf.get_class_prefix(key_value, generic_method)
method_reporting = '"dummy_report_' + str(report_unique_id) + '" usebundle => ' + ncf.generate_reporting_context(generic_method, method_call)
report_unique_id += 1
class_parameter = ncf.generate_reporting_class_parameter(generic_method, method_call)
if not "cfengine-community" in generic_method["agent_support"]:
message = "'"+generic_method["name"]+"' method is not available on cfengine based agent, skip"
logger_call = get_logger_call(message, class_prefix, class_parameter)
content.append(' any::')
content.append(' "dummy_report" usebundle => _classes_noop("'+class_prefix+'");')
content.append(' ' + method_reporting + ';')
content.append(' ' + logger_call+';')
elif method_call['class_context'] != "any":
# escape double quote in key value
regex_quote = re.compile(r'(?<!\\)"', flags=re.UNICODE )
escaped_key_value = regex_quote.sub('\\"', key_value)
message = generic_method['name'] + ' ' + escaped_key_value + ' if ' + method_call['class_context']
logger_rudder_call = get_logger_call(message, class_prefix, class_parameter)
# Always add an empty line for readability
content.append('')
if not "$" in method_call['class_context']:
content.append(' !('+method_call['class_context']+')::')
content.append(' "dummy_report" usebundle => _classes_noop("'+class_prefix+'");')
content.append(' ' + method_reporting + ';')
content.append(' ' + logger_rudder_call + ';')
else:
class_context = ncf.canonify_class_context(method_call['class_context'])
content.append(' "dummy_report" usebundle => _classes_noop("'+class_prefix+'"),')
content.append(' ifvarclass => concat("!('+class_context+')");')
content.append(' ' + method_reporting + ',')
content.append(' ifvarclass => concat("!('+class_context+')");')
content.append(' ' + logger_rudder_call + ',')
content.append(' ifvarclass => concat("!('+class_context+')");')
content.append('}')
# Join all lines with \n to get a pretty CFEngine file
result = '\n'.join(content)+"\n"
return result
cfengine_versions = list(min_version.keys())
cfengine_versions.sort(key=StrictVersion)
for cfengine_version in cfengine_versions:
if StrictVersion(canonified_version) <= StrictVersion(cfengine_version):
return min_version[cfengine_version]
return False
if __name__ == '__main__':
# Get CFEngine <=> Rudder versions
versions = get_min_versions()
# Get all generic methods
generic_methods = ncf.get_all_generic_methods_metadata()["data"]["generic_methods"]
categories = {}
for method_name in sorted(generic_methods.keys()):
category_name = method_name.split('_',1)[0]
generic_method = generic_methods[method_name]
if (category_name in categories):
categories[category_name].append(generic_method)
else:
categories[category_name] = [generic_method]
content = []
for category in sorted(categories.keys()):
content.append("\n*****\n")
content.append('\n### '+category.title())
def generate_rudder_reporting(technique):
"""Generate complementary reporting needed for Rudder in rudder_reporting.st file"""
# Get all generic methods
generic_methods = ncf.get_all_generic_methods_metadata(
alt_path='/var/rudder/configuration-repository/ncf'
)['data']['generic_methods']
content = []
content.append('bundle agent ' + technique['bundle_name'] +
'_rudder_reporting')
content.append('{')
content.append(' vars:')
content.append(
' "promisers" slist => { @{this.callers_promisers}, cf_null }, policy => "ifdefined";'
)
content.append(
' "class_prefix" string => canonify(join("_", "promisers"));')
content.append(' "args" slist => { };')
content.append('')
content.append(' methods:')
for method_call in technique["method_calls"]:
method_name = method_call['method_name']
if method_call['method_name'].startswith("_"):
continue
generic_method = generic_methods[method_name]
key_value = ncf.get_key_value(method_call, generic_method)
class_prefix = ncf.get_class_prefix(key_value, generic_method)
if not "cfengine-community" in generic_method["agent_support"]:
message = "'" + generic_method[
"name"] + "' method is not available on cfengine based agent, skip"
logger_call = ncf.get_logger_call(message, class_prefix)
content.append(' any::')
content.append(' "dummy_report" usebundle => _classes_noop("' +
class_prefix + '");')
content.append(' ' + logger_call + ';')
elif method_call['class_context'] != "any":
# escape double quote in key value
regex_quote = re.compile(r'(?<!\\)"', flags=re.UNICODE)
escaped_key_value = regex_quote.sub('\\"', key_value)
message = generic_method[
'name'] + ' ' + escaped_key_value + ' if ' + method_call[
'class_context']
logger_rudder_call = ncf.get_logger_call(message, class_prefix)
# Always add an empty line for readability
content.append('')
if not "$" in method_call['class_context']:
content.append(' !(' + method_call['class_context'] + ')::')
content.append(
' "dummy_report" usebundle => _classes_noop("' +
class_prefix + '");')
content.append(' ' + logger_rudder_call + ';')
else:
class_context = ncf.canonify_class_context(
method_call['class_context'])
content.append(
' "dummy_report" usebundle => _classes_noop("' +
class_prefix + '"),')
content.append(' ifvarclass => concat("!(' +
class_context + ')");')
content.append(' ' + logger_rudder_call + ',')
content.append(' ifvarclass => concat("!(' +
class_context + ')");')
content.append('}')
# Join all lines with \n to get a pretty CFEngine file
result = '\n'.join(content) + "\n"
return result
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Usage: ./ncf_doc.py
#
# This is a Python module to generate documentation from generic methods in ncf
import ncf
if __name__ == '__main__':
# Get all generic methods
generic_methods = ncf.get_all_generic_methods_metadata()
categories = {}
for method_name in sorted(generic_methods.iterkeys()):
category_name = method_name.split('_',1)[0]
generic_method = generic_methods[method_name]
if (category_name in categories):
categories[category_name].append(generic_method)
else:
categories[category_name] = [generic_method]
content = []
html_content = []
content.append("Title: Reference")
content.append("slugs: reference")