def mavencleancompile(modules_location):
"""
Executes mvn clean and mvn compile in the given modules_location.
"""
LOGGER.info("Doing maven clean in %s." % modules_location)
output = run_asyncloop("mvn clean", startpath=modules_location)
LOGGER.debug(output)
LOGGER.info("Doing maven compile in %s." % modules_location)
output = run_asyncloop("mvn compile", startpath=modules_location)
LOGGER.debug(output)
def maven_clean_compile(repo_location, repository):
"""
Executes mvn clean and mvn compile in the given modules_location.
"""
repoloc = os.path.join(repo_location, repository)
logger.info("Doing maven clean compile in %s." % repoloc)
output = run_asyncloop("mvn clean compile", startpath=repoloc)
logger.debug(output)
def convertpodtomarkdown(podfile, outputfile):
"""
Takes a podfile and converts it to a markdown with the help of pod2markdown.
"""
LOGGER.debug("Running pod2markdown on %s." % podfile)
output = run_asyncloop("pod2markdown %s" % podfile)
LOGGER.debug("writing output to %s." % outputfile)
LOGGER.debug(output)
with open(outputfile, "w") as fih:
fih.write(output[1])
def makeResult(self, profile, mode=None):
"""Compile the profile from SERVICE and run the template toolkit on it"""
tmpdir = self.tmpdir
self.pancout = None
self.json2ttout = None
includepaths = [self.PROFILEPATH, self.extradir, self.TEMPLATE_LIBRARY_CORE]
cmd = [
'panc',
'--formats' , 'json',
'--include-path', os.pathsep.join(includepaths),
'--output-dir', tmpdir,
os.path.join(self.PROFILEPATH, "%s.pan" % profile)
]
ec, out = run_asyncloop(cmd)
jsonfile = os.path.join(tmpdir, "%s.json" % profile)
if not os.path.exists(jsonfile):
logging.debug("No json file found for service %s and profile %s. cmd %s output %s" % (self.SERVICE, profile, cmd, out))
self.pancout = out
return
if self.SHOWJSON:
print "profile %s JSON:\n%s" % (profile, open(jsonfile).read())
if mode is None:
mode = ['--unittest']
cmd = [
'perl', self.JSON2TT,
'--json', jsonfile,
'--includepath', os.path.dirname(self.METACONFIGPATH)
] + mode
ec, out = run_asyncloop(cmd)
if ec > 0:
logging.debug("json2tt exited with non-zero ec %s: %s" % (ec, out))
self.json2ttout = out
return
self.result = out
if self.SHOWTT:
print "profile %s TT output:\n%s" % (profile, out)
def makeResult(self, profile, mode=None):
"""Compile the profile from SERVICE and run the template toolkit on it"""
tmpdir = self.tmpdir
self.pancout = None
self.json2ttout = None
includepaths = [self.PROFILEPATH, self.extradir, self.TEMPLATE_LIBRARY_CORE]
cmd = [
'panc',
'--formats' , 'json',
'--include-path', os.pathsep.join(includepaths),
'--output-dir', tmpdir,
os.path.join(self.PROFILEPATH, "%s.pan" % profile)
]
ec, out = run_asyncloop(cmd)
jsonfile = os.path.join(tmpdir, "%s.json" % profile)
if not os.path.exists(jsonfile):
logging.debug("No json file found for service %s and profile %s. cmd %s output %s" % (self.SERVICE, profile, cmd, out))
self.pancout = out
return
if self.SHOWJSON:
print "profile %s JSON:\n%s" % (profile, open(jsonfile).read())
if mode is None:
mode = ['--unittest']
cmd = [
'perl', self.JSON2TT,
'--json', jsonfile,
'--includepath', os.path.dirname(self.METACONFIGPATH)
] + mode
ec, out = run_asyncloop(cmd)
if ec > 0:
logging.debug("json2tt exited with non-zero ec %s: %s" % (ec, out))
self.json2ttout = out
return
self.result = out
if self.SHOWTT:
print "profile %s TT output:\n%s" % (profile, out)
def main(indices):
"""The main, only test the indices passed"""
prep_grok()
input, results = get_data()
if indices:
input = [input[idx] for idx in indices]
results = [results[idx] for idx in indices]
ec, stdout = run_asyncloop(cmd=LOGSTASH_CMD, input="\n".join(input + ['']))
output = process(stdout, len(input))
test(output, input, results)
def build_annotations(file, basedir, outputdir):
"""Build pan annotations."""
panccommand = ["panc-annotations", "--output-dir", outputdir, "--base-dir", basedir]
panccommand.append(file)
logger.debug("Running %s." % panccommand)
ec, output = output = run_asyncloop(panccommand)
logger.debug(output)
if ec == 0 and os.path.exists(os.path.join(outputdir, "%s.annotation.xml" % file)):
return True
else:
logger.warning("Something went wrong running '%s'." % panccommand)
return False
def markdown_from_perl(podfile):
"""
Take a perl file and converts it to a markdown with the help of pod2markdown.
Returns True if pod2markdown worked, False if it failed.
"""
logger.info("Making markdown from perl: %s." % podfile)
ec, output = run_asyncloop("pod2markdown %s" % podfile)
logger.debug(output)
if ec != 0 or output == "\n":
logger.warning("pod2markdown failed on %s." % podfile)
return None
else:
return output
def markdown_from_perl(podfile):
"""
Take a perl file and converts it to a markdown with the help of pod2markdown.
Returns True if pod2markdown worked, False if it failed.
"""
logger.info("Making markdown from perl: %s." % podfile)
ec, output = run_asyncloop("pod2markdown %s" % podfile)
logger.debug(output)
if ec != 0 or output == "\n":
logger.warning("pod2markdown failed on %s." % podfile)
return None
else:
return output
def build_annotations(pfile, basedir, outputdir):
"""Build pan annotations."""
panccommand = [
"panc-annotations", "--output-dir", outputdir, "--base-dir", basedir
]
panccommand.append(pfile)
logger.debug("Running %s." % panccommand)
ec, output = output = run_asyncloop(panccommand)
logger.debug(output)
if ec == 0 and os.path.exists(
os.path.join(outputdir, "%s.annotation.xml" % pfile)):
return True
else:
logger.warning("Something went wrong running '%s'." % panccommand)
return False
def convertpodtomarkdown(podfile, outputfile, title):
"""
Takes a podfile and converts it to a markdown with the help of pod2markdown.
"""
LOGGER.debug("Running pod2markdown on %s." % podfile)
output = run_asyncloop("pod2markdown %s" % podfile)
LOGGER.debug("writing output to %s." % outputfile)
LOGGER.debug(output)
fih = open(outputfile, "w")
fih.write("---\n")
fih.write("layout: article\n")
fih.write("title: %s\n" % title)
fih.write("category: documentation\n")
fih.write("---\n")
fih.write(output[1])
fih.close()
"""The main, only test the indices passed"""
prep_grok()
input, results = get_data()
if indices:
for indx in indices:
_log.debug("Test index %d => input: %s" % (indx, input[indx]))
_log.debug("Test index %d => results: %s" % (indx, results[indx]))
try:
input = [input[idx] for idx in indices]
results = [results[idx] for idx in indices]
except IndexError, e:
_log.error('Provided indices %s exceed avail data items %s' % (indices, len(input)))
sys.exit(1)
ec, stdout = run_asyncloop(cmd=LOGSTASH_CMD+[cfg_file], input="\n".join(input + ['']))
output = process(stdout, len(input))
test(output, input, results)
if __name__ == '__main__':
opts = {
"last": ("Only test last data entry", None, "store_true", False, 'L'),
"first": ("Only test first data entry", None, "store_true", False, 'F'),
"entries": ("Indices of data entries to test", "strlist", "store", None, 'E'),
"logstash-version": ("Logstash version to test with", None, "store", DEFAULT_LOGSTASH_VERSION, 'V'),
}
go = simple_option(opts)
indices = None
if go.options.first:
indices = [0]
def test_simple_asyncloop(self):
ec, output = run_asyncloop([sys.executable, SCRIPT_SIMPLE, 'shortsleep'])
self.assertEqual(ec, 0)
self.assertTrue('shortsleep' in output.lower())
def test_simple_asyncloop(self):
ec, output = run_asyncloop(
[sys.executable, SCRIPT_SIMPLE, 'shortsleep'])
self.assertEqual(ec, 0)
self.assertTrue('shortsleep' in output.lower())
def test_simple_asyncloop(self):
ec, output = run_asyncloop([SCRIPT_SIMPLE, "shortsleep"])
self.assertEqual(ec, 0)
self.assertTrue("shortsleep" in output.lower())
def maven_clean_compile(location):
"""Execute mvn clean and mvn compile in the given modules_location."""
logger.info("Doing maven clean compile in %s." % location)
ec, output = run_asyncloop("mvn clean compile", startpath=location)
logger.debug(output)
return ec
prep_grok()
input, results = get_data()
if indices:
for indx in indices:
_log.debug("Test index %d => input: %s" % (indx, input[indx]))
_log.debug("Test index %d => results: %s" % (indx, results[indx]))
try:
input = [input[idx] for idx in indices]
results = [results[idx] for idx in indices]
except IndexError, e:
_log.error('Provided indices %s exceed avail data items %s' %
(indices, len(input)))
sys.exit(1)
ec, stdout = run_asyncloop(cmd=LOGSTASH_CMD + [cfg_file],
input="\n".join(input + ['']))
output = process(stdout, len(input))
test(output, input, results)
if __name__ == '__main__':
opts = {
"last": ("Only test last data entry", None, "store_true", False, 'L'),
"first":
("Only test first data entry", None, "store_true", False, 'F'),
"entries":
("Indices of data entries to test", "strlist", "store", None, 'E'),
"logstash-version": ("Logstash verison to test with", None, "store",
DEFAULT_LOGSTASH_VERSION, 'V'),
}
def create_md_from_pan(source, comppath):
"""
Takes a pan schema, creates the pan annotations and parses them to markdown.
"""
modname = os.path.basename(os.path.split(source)[0])
mdfile = modname + "::schema.md"
tmpdir = tempfile.mkdtemp()
logger.debug("Temporary directory: %s" % tmpdir)
panccommand = ["panc-annotations", "--output-dir", tmpdir, "--base-dir"]
panccommand.extend(os.path.split(source))
output = run_asyncloop(panccommand)
logger.debug(output)
namespace = "{http://quattor.org/pan/annotations}"
tpl = "schema.pan.annotation.xml"
xml = etree.parse(os.path.join(tmpdir, tpl))
root = xml.getroot()
mdtext = []
mdtext.append("# Types\n")
for stype in root.findall('%stype' % namespace):
name = stype.get('name')
mdtext.append("- /software/%s/%s" % (modname, name))
for doc in stype.findall(".//%sdesc" % namespace):
mdtext.append("%s- decription: %s" % (" "*4, doc.text))
for field in stype.findall(".//%sfield" % namespace):
mdtext.append("%s- /software/%s/%s/%s" % (" "*4, modname, name, field.get('name')))
required = field.get('required')
if required == "true":
mdtext.append("%s- required" % (" "*8))
else:
mdtext.append("%s- optional" % (" "*8))
for basetype in field.findall(".//%sbasetype" % namespace):
fieldtype = basetype.get('name')
mdtext.append("%s- type: %s" % (" "*8, fieldtype))
if fieldtype == "long" and basetype.get('range'):
fieldrange = basetype.get('range')
mdtext.append("%s- range: %s" % (" "*8, fieldrange))
mdtext.append("\n")
deffunctions = root.findall('%sfunction' % namespace)
if deffunctions:
mdtext.append("\n# Functions\n")
root.findall('%sfunction' % namespace)
for fnname in deffunctions:
name = fnname.get('name')
mdtext.append("- %s" % name)
for doc in fnname.findall(".//%sdesc" % namespace):
mdtext.append(" description: %s " % doc.text)
for arg in fnname.findall(".//%sarg" % namespace):
mdtext.append("- arg: %s " % arg.text)
with open(os.path.join(comppath, mdfile), "w") as fih:
fih.write("\n".join(mdtext))
logger.debug("Removing temporary directory: %s" % tmpdir)
shutil.rmtree(tmpdir)
return mdfile
def maven_clean_compile(location):
"""Execute mvn clean and mvn compile in the given modules_location."""
logger.info("Doing maven clean compile in %s." % location)
ec, output = run_asyncloop("mvn clean compile", startpath=location)
logger.debug(output)
return ec