def test_check_file(self):
"""Test check_file()"""
# function should return false when file doesn't exist
self.assertFalse(check_file('nofile.csv'))
self.assertFalse(check_file(''))
# function should throw an error when no filename is set
self.assertRaises(TypeError, check_file)
# function should return true if file exists
self.assertTrue(check_file(constants.TEST_SAMPLE_TIMESTAMP_FILE))
def test_check_file(self):
"""Test check_file()"""
# function should return false when file doesn't exist
self.assertFalse(check_file('nofile.csv'))
self.assertFalse(check_file(''))
# function should throw an error when no filename is set
self.assertRaises(TypeError, check_file)
# function should return true if file exists
self.assertTrue(check_file(constants.TEST_SAMPLE_TIMESTAMP_FILE))
def test_generate(self):
"""Test generate()"""
self.trend.builds = ['10', '11.1', '#3']
self.trend.stages = {
'stage1': [4, 3, 2],
'stage2': [5, 6, 7],
'stage3': [10, 11, 12],
'stage4': [1, 0, 3],
'stage5': [0, 6, 0]
}
self.assertFalse(tools.check_file(TEST_TREND_FILE))
self.trend.generate(TEST_TREND_FILE)
self.assertTrue(tools.check_file(TEST_TREND_FILE))
def test_generate(self):
"""Test generate()"""
self.trend.builds = ['10', '11.1', '#3']
self.trend.stages = {
'stage1': [4, 3, 2],
'stage2': [5, 6, 7],
'stage3': [10, 11, 12],
'stage4': [1, 0, 3],
'stage5': [0, 6, 0]
}
self.assertFalse(tools.check_file(TEST_TREND_FILE))
self.trend.generate(TEST_TREND_FILE)
self.assertTrue(tools.check_file(TEST_TREND_FILE))
def load_config_file(self, config_file):
"""
Load settings from a config file.
Parameters :
- config_file : name of the config file
"""
if not check_file(config_file):
return False
with open(config_file, 'r') as file_stream:
config = yaml.load(file_stream)
if "buildtimetrend" in config and \
is_dict(config["buildtimetrend"]):
self.settings.add_items(config["buildtimetrend"])
set_loglevel(self.get_setting("loglevel"))
# set Keen.io settings
if "keen" in config:
if "project_id" in config["keen"]:
keen.project_id = config["keen"]["project_id"]
if "write_key" in config["keen"]:
keen.write_key = config["keen"]["write_key"]
if "read_key" in config["keen"]:
keen.read_key = config["keen"]["read_key"]
if "master_key" in config["keen"]:
keen.master_key = config["keen"]["master_key"]
return True
def load_config_file(self, config_file):
"""
Load settings from a config file.
Parameters :
- config_file : name of the config file
"""
if not check_file(config_file):
return False
with open(config_file, 'r') as file_stream:
config = yaml.load(file_stream)
if "buildtimetrend" in config and \
is_dict(config["buildtimetrend"]):
self.settings.add_items(config["buildtimetrend"])
set_loglevel(self.get_setting("loglevel"))
# set Keen.io settings
if "keen" in config:
if "project_id" in config["keen"]:
keen.project_id = config["keen"]["project_id"]
if "write_key" in config["keen"]:
keen.write_key = config["keen"]["write_key"]
if "read_key" in config["keen"]:
keen.read_key = config["keen"]["read_key"]
if "master_key" in config["keen"]:
keen.master_key = config["keen"]["master_key"]
return True
def test_generate_config_file(self, get_cfg_str_func):
"""Test dashboard.generate_config_file()"""
# set config file path
Settings().add_setting("dashboard_configfile",
constants.DASHBOARD_TEST_CONFIG_FILE)
# check if configfile exists
self.assertFalse(check_file(constants.DASHBOARD_TEST_CONFIG_FILE))
# generate config file with empty repo name
self.assertRaises(TypeError, dashboard.generate_config_file)
# generate config file with empty repo name
self.assertTrue(dashboard.generate_config_file(None))
self.assertTrue(check_file(constants.DASHBOARD_TEST_CONFIG_FILE))
# check if mock was called with correct parameters
args, kwargs = get_cfg_str_func.call_args
self.assertEqual(args, (None, ))
self.assertDictEqual(kwargs, {})
# generate config file
self.assertTrue(dashboard.generate_config_file("test/repo3"))
self.assertTrue(check_file(constants.DASHBOARD_TEST_CONFIG_FILE))
# check if mock was called with correct parameters
args, kwargs = get_cfg_str_func.call_args
self.assertEqual(args, ("test/repo3", ))
self.assertDictEqual(kwargs, {})
# test generated config file contents
with open(constants.DASHBOARD_TEST_CONFIG_FILE, 'r') as config_file:
self.assertEqual("var config = {'projectName': 'test/repo3'};\n",
next(config_file))
self.assertEqual("var keenConfig = {'projectId': '1234abcd'};",
next(config_file))
def parse_job_log_file(self, filename):
"""
Open a Travis CI log file and parse it.
Parameters :
- filename : filename of Travis CI log
Returns false if file doesn't exist, true if it was read successfully.
"""
# load timestamps file
if not tools.check_file(filename):
return False
# read timestamps, calculate stage duration
with open(filename, 'rb') as file_stream:
self.parse_job_log_stream(file_stream)
return True
def parse_job_log_file(self, filename):
'''
Open a Travis CI log file and parse it.
Parameters :
- filename : filename of Travis CI log
Returns false if file doesn't exist, true if it was read successfully.
'''
# load timestamps file
if not check_file(filename):
return False
# read timestamps, calculate stage duration
with open(filename, 'rb') as file_stream:
self.parse_job_log_stream(file_stream)
return True
def read_csv(self, csv_filename):
'''
Gathers timestamps from a csv file and calculates stage duration.
Parameters :
- csv_filename : csv filename containing timestamps
Returns false if file doesn't exist, true if it was read successfully.
'''
# load timestamps file
if not check_file(csv_filename):
return False
# read timestamps, calculate stage duration
with open(csv_filename, 'rb') as csv_data:
timestamps = csv.reader(csv_data, delimiter=',', quotechar='"')
self.parse_timestamps(timestamps)
return True
def gather_data(self, result_file):
"""
Get buildtime data from an xml file.
Parameters
- result_file : xml file containing the buildtime data
"""
# load buildtimes file
if check_file(result_file):
root_xml = etree.parse(result_file).getroot()
else:
return False
index = 0
# print content of buildtimes file
for build_xml in root_xml:
build_id = build_xml.get('build')
job_id = build_xml.get('job')
if job_id is None and build_id is None:
build_name = "#{0:d}".format(index + 1)
elif job_id is not None:
build_name = job_id
else:
build_name = build_id
self.builds.append(build_name)
# add 0 to each existing stage, to make sure that
# the indexes of each value
# are correct, even if a stage does not exist in a build
# if a stage exists, the zero will be replaced by its duration
for stage in self.stages:
self.stages[stage].append(0)
# add duration of each stage to stages list
for build_child in build_xml:
if build_child.tag == 'stages':
stage_count = len(build_child)
self.parse_xml_stages(build_child, index)
logger.debug("Build ID : %s, Job : %s, stages : %d",
build_id, job_id, stage_count)
index += 1
return True
def gather_data(self, result_file):
"""
Get buildtime data from an xml file.
Parameters
- result_file : xml file containing the buildtime data
"""
# load buildtimes file
if check_file(result_file):
root_xml = etree.parse(result_file).getroot()
else:
return False
index = 0
# print content of buildtimes file
for build_xml in root_xml:
build_id = build_xml.get('build')
job_id = build_xml.get('job')
if job_id is None and build_id is None:
build_name = "#{0:d}".format(index + 1)
elif job_id is not None:
build_name = job_id
else:
build_name = build_id
self.builds.append(build_name)
# add 0 to each existing stage, to make sure that
# the indexes of each value
# are correct, even if a stage does not exist in a build
# if a stage exists, the zero will be replaced by its duration
for stage in self.stages:
self.stages[stage].append(0)
# add duration of each stage to stages list
for build_child in build_xml:
if build_child.tag == 'stages':
stage_count = len(build_child)
self.parse_xml_stages(build_child, index)
logger.debug("Build ID : %s, Job : %s, stages : %d", build_id,
job_id, stage_count)
index += 1
return True
def generate_overview_config_file(repo):
'''
Generates a config file for the overview HTML file that contains the
graphs generated by Keen.io
'''
logger = get_logger()
if not ("BUILD_TREND_SAMPLE_CONFIGFILE" in os.environ
and "BUILD_TREND_CONFIGFILE" in os.environ
and "KEEN_PROJECT_ID" in os.environ):
logger.error("Trends overview config file was not created")
return
sample_filename = os.getenv("BUILD_TREND_SAMPLE_CONFIGFILE")
config_file = os.getenv("BUILD_TREND_CONFIGFILE")
keen_project_id = os.getenv("KEEN_PROJECT_ID")
# check if sample config file exists
if not check_file(sample_filename):
return
# generate read key
read_key = keen_io_generate_read_key(repo)
# list of string pairs to be replaced (search string : new string)
replacements = {
'keen_project_id': str(keen_project_id),
'keen_read_key': str(read_key),
'project_name': str(repo)
}
with open(sample_filename, 'rb') as infile, \
open(config_file, 'w') as outfile:
for line in infile:
for src, target in replacements.iteritems():
line = line.replace(src, target)
outfile.write(line)
logger.info("Created trends overview config file")
def test_generate_config_file_fails(self):
"""Test dashboard.generate_config_file() if creation fails"""
# set config file path
Settings().add_setting("dashboard_configfile",
constants.DASHBOARD_TEST_CONFIG_FILE)
# check if configfile exists
self.assertFalse(check_file(constants.DASHBOARD_TEST_CONFIG_FILE))
# init mock
patcher = mock.patch('buildtimetrend.tools.check_file',
return_value=False)
check_file_func = patcher.start()
# generation should return false
self.assertFalse(dashboard.generate_config_file("test/repo4"))
# check if mock was called with correct parameters
args, kwargs = check_file_func.call_args
self.assertEqual(args, (constants.DASHBOARD_TEST_CONFIG_FILE, ))
self.assertDictEqual(kwargs, {})
patcher.stop()
def modify_index(file_original, file_modified):
"""
Modify html file for Buildtime Trend as a Service.
Adjust paths to 'assets' :
the relative path is changed to an absolute path.
Parameters:
- file_original : Path of the original file
- file_modified : Path of the modified file hosted on the service
"""
if not file_is_newer(file_modified, file_original):
with open(file_original, 'r') as infile, \
open(file_modified, 'w') as outfile:
for line in infile:
line = line.replace("assets", ASSETS_URL)
outfile.write(line)
if check_file(file_modified):
logger.info("Created index service file : %s", file_modified)
return True
else:
return False
def log_build_native(build):
'''Store build data in xml format'''
# import dependency
from lxml import etree
# load previous buildtimes file, or create a new xml root
if check_file(RESULT_FILE):
try:
root_xml = etree.parse(RESULT_FILE).getroot()
except etree.XMLSyntaxError:
get_logger().error('XML format invalid : a new file is created,'
' corrupt file is discarded')
root_xml = etree.Element("builds")
else:
root_xml = etree.Element("builds")
# add build data to xml
root_xml.append(build.to_xml())
# write xml to file
with open(RESULT_FILE, 'wb') as xmlfile:
xmlfile.write(etree.tostring(
root_xml, xml_declaration=True,
encoding='utf-8', pretty_print=True))
def tearDown():
"""Clean up after tests"""
if (tools.check_file(TEST_TREND_FILE)):
os.remove(TEST_TREND_FILE)
def tearDown():
"""Clean up after tests"""
if (check_file(constants.DASHBOARD_TEST_CONFIG_FILE)):
os.remove(constants.DASHBOARD_TEST_CONFIG_FILE)
def tearDown():
"""Clean up after tests"""
if (tools.check_file(TEST_TREND_FILE)):
os.remove(TEST_TREND_FILE)
def index(self):
"""Index page."""
if check_file(self.file_index):
return open(self.file_index)
else:
raise cherrypy.HTTPError(404, "File not found")
def log_build_native(build):
'''Store build data in xml format'''
# import dependency
from lxml import etree
# load previous buildtimes file, or create a new xml root
if check_file(RESULT_FILE):
try:
root_xml = etree.parse(RESULT_FILE).getroot()
except etree.XMLSyntaxError:
get_logger().error('XML format invalid : a new file is created,'
' corrupt file is discarded')
root_xml = etree.Element("builds")
else:
root_xml = etree.Element("builds")
# add build data to xml
root_xml.append(build.to_xml())
# write xml to file
with open(RESULT_FILE, 'wb') as xmlfile:
xmlfile.write(etree.tostring(
root_xml, xml_declaration=True,
encoding='utf-8', pretty_print=True))
if __name__ == "__main__":
# only run analysis if timestampfile is present
if check_file(TIMESTAMP_FILE):
analyse(sys.argv)