def run(self):
own_lock = False
build_system_name = self.identifyBuildSystemName()
self.proclaim_existence(build_system_name)
own_lock = self.acquireLock()
try:
for config_file in self.config_file_names:
config_file_path = self.find_config_file(config_file)
if not config_file_path:
raise ConfigurationError(
"No config file for '%s' found in the config subdir" %
config_file)
lf_name = "log/%s.log" % config_file.replace(
'.yml', '').replace('_config', '')
self.log = ActivityLogger(lf_name)
logAllExceptions(True, self.log)
self._operateService(config_file_path)
except Exception as msg:
self.log.error(msg)
finally:
try:
if own_lock: self.releaseLock()
except Exception as msg:
raise OperationalError(
"ERROR: unable to remove lock file '%s', %s" %
(LOCK_FILE, msg))
self.log.info('run completed')
def __init__(self, args):
"""
An instance of this class is responsible for basic command line parsing,
pulling off the name of the target config file.
Once that item is taken care of, the instance obtains an instance of a
Konfabulator, a AuxiliaryClassLoader instance and and instance of an BLDConnector.
The Konfabulator is delegated the task of reading the config file, the AuxiliaryClassLoader
is delegated the task of pulling in any Extension classes.
These instances are then provided to the BuildConnector instance.
This instance then directs the BuildConnector instance to obtain unrecorded builds
from the target Build/CI system and reflect them in the Agile Central server.
"""
if len(args) < 1:
problem = (
"Insufficient command line args, must be at least a config file name."
)
raise ConfigurationError(problem)
self.default_log_file_name = True
# this only allows a logfile spec of --logfile=<some_file_name>
# TODO: consider allowing additional syntax logfile=foo.log or --logfile foo.log
log_spec = [arg for arg in args if arg.startswith('--logfile=')]
if log_spec:
self.logfile_name = log_spec[0].replace('--logfile=', '')
self.default_log_file_name = False
for spec in log_spec:
args = [arg for arg in args if arg != spec]
self.config_file_names = args
if self.default_log_file_name: # set it to first config file minus any '_config.yml' portion
self.first_config = self.config_file_names[0]
self.logfile_name = "log/%s.log" % self.first_config.replace(
'.yml', '').replace('_config', '')
try:
if not os.path.exists('log'):
os.makedirs('log')
except Exception as msg:
sys.stderr.write(
"Unable to locate or create the log sub-directory, %s\n" %
msg)
raise Exception
self.log = ActivityLogger(self.logfile_name)
logAllExceptions(True, self.log)
self.preview = False
self.connector = None
self.extension = {}
def test_tab_yml():
logger = ActivityLogger('log/bad_tab.log')
expectedErrPattern = "Your config file contains tab characters which are not allowed in a YML file."
with pytest.raises(Exception) as excinfo:
konf = Konfabulator('config/bad_tab.yml', logger)
actualErrVerbiage = excinfo.value.args[0]
assert re.search(expectedErrPattern, actualErrVerbiage)
def test_config_defaults():
#config_file = ('defaults.yml')
logger = ActivityLogger('log/defaults.log')
konf = Konfabulator('config/defaults.yml', logger)
jenk_conf = konf.topLevel('Jenkins')
ac_conf = konf.topLevel('AgileCentral')
srv_config = konf.topLevel('Service')
assert not ac_conf.get('Server', None)
assert not jenk_conf.get('Server', None)
# runner = BuildConnectorRunner([config_file])
# runner.run()
agicen = bsh.AgileCentralConnection(konf.topLevel('AgileCentral'), logger)
agicen.other_name = 'Jenkins'
agicen.project_name = jenk_conf['AgileCentral_DefaultBuildProject']
agicen.connect()
assert agicen.server == 'rally1.rallydev.com'
assert not agicen.proxy
jc = bsh.JenkinsConnection(jenk_conf, logger)
jc.connect()
assert jc.server == 'coyotepair.ca.com'
assert jc.port == 8080
assert jc.protocol == 'http'
def setup_test_config(filename,
jenkins_structure='DEFAULT_JENKINS_STRUCTURE',
services='DEFAULT_SERVICES'):
filename = inflate_config_file(filename, jenkins_structure, services)
logger = ActivityLogger('test.log')
tkonf = TestKonfabulator(filename, logger)
return logger, tkonf
def test_without_project():
config_file = ('missing-project.yml')
ymlfile = open("config/{}".format(config_file), 'r')
y = yaml.load(ymlfile)
logger = ActivityLogger('log/missing-project.log')
logger.setLevel('DEBUG')
logAllExceptions(True, logger)
konf = Konfabulator('config/missing-project.yml', logger)
jenk_conf = konf.topLevel('Jenkins')
ac_conf = konf.topLevel('AgileCentral')
#expectedErrPattern = 'The Jenkins section of the config is missing AgileCentral_DefaultBuildProject property'
expectedErrPattern = 'The Jenkins section of the config is missing a value for AgileCentral_DefaultBuildProject property'
with pytest.raises(Exception) as excinfo:
bc = bsh.BLDConnector(konf, logger)
actualErrVerbiage = excinfo.value.args[0]
assert re.search(expectedErrPattern, actualErrVerbiage) is not None
assert excinfo.typename == 'ConfigurationError'
def test_without_project():
config_file = ('missing-project.yml')
ymlfile = open("config/{}".format(config_file), 'r')
y = yaml.load(ymlfile)
logger = ActivityLogger('log/missing-project.log')
logger.setLevel('DEBUG')
logAllExceptions(True, logger)
konf = Konfabulator('config/missing-project.yml', logger)
jenk_conf = konf.topLevel('Jenkins')
ac_conf = konf.topLevel('AgileCentral')
#expectedErrPattern = 'The Jenkins section of the config is missing AgileCentral_DefaultBuildProject property'
expectedErrPattern = 'The Jenkins section of the config is missing a value for AgileCentral_DefaultBuildProject property'
with pytest.raises(Exception) as excinfo:
bc = bsh.BLDConnector(konf, logger)
actualErrVerbiage = excinfo.value.args[0]
assert re.search(expectedErrPattern, actualErrVerbiage) is not None
assert excinfo.typename == 'ConfigurationError'
def test_bad_yml():
logger = ActivityLogger('log/bad_yml.log')
expectedErrPattern = "The file does not contain consistent indentation for the sections and section contents"
unexpectedErrPattern = "Oh noes!"
with pytest.raises(Exception) as excinfo:
konf = Konfabulator('config/bad_yml.yml', logger)
actualErrVerbiage = excinfo.value.args[0]
assert re.search(expectedErrPattern, actualErrVerbiage)
assert not re.search(unexpectedErrPattern, actualErrVerbiage)
def connect_to_jenkins(config_file):
config_file = "config/{}".format(config_file)
jenk_conf = {}
with open(config_file, 'r') as cf:
content = cf.read()
all_conf = yaml.load(content)
jenk_conf = all_conf['JenkinsBuildConnector']['Jenkins']
jc = bsh.JenkinsConnection(jenk_conf, ActivityLogger('inventory.log'))
return jc
def connect_to_ac(config_file):
logger = ActivityLogger('kublakhan.log')
konf = Konfabulator('config/buildorama.yml', logger)
jenk_conf = konf.topLevel('Jenkins')
ac_conf = konf.topLevel('AgileCentral')
ac_conf['Project'] = jenk_conf[
'AgileCentral_DefaultBuildProject'] # leak proj from jenkins section to ac section
agicen = AgileCentralConnection(ac_conf, logger)
agicen.other_name = 'Jenkins'
agicen.connect()
return agicen
def main(args):
conf = yaml.load(CONFIG_CHUNK)
jenkconf = conf['Jenkins']
#pprint(jenkconf)
#print("=" * 80)
#print("")
logger = ActivityLogger("jenk.log", policy='calls', limit=1000)
jenkins = JenkinsConnection(jenkconf, logger)
started = time.time()
try:
jenkins.connect()
except Exception as ex:
# does ex include the following text?
# Max retries exceeded with url: /manage .* nodename nor servname provided, or not known'
print(sys.exc_info()[0]) # 0 is the Exception instance
print(sys.exc_info()[1]) # 1 is the Exception text
ref_timestamp = (2016, 6, 1, 0, 0, 0, 5, 0, -1)
#top_level_folders = sorted(jenkins.view_folders['All'].keys())
#for folder_name in top_level_folders:
# folder = jenkins.view_folders['All'][folder_name]
# print(folder.info())
builds = jenkins.getRecentBuilds(ref_timestamp)
finished = time.time()
for tank_and_project in sorted(builds):
container, project = tank_and_project.split('::', 1)
print("Jenkins Container: %s" % container)
print("AgileCentral Project: %s" % project)
for job in builds[tank_and_project]:
print(" Job: %s" % job)
build_results = builds[tank_and_project][job]
for build in build_results:
print("%s%s" % (" " * 8, build))
print("")
#for view_and_project in builds:
# view, project = view_and_project.split('::', 1)
# print("Jenkins View: %s" % view)
# print("AgileCentral Project: %s" % project)
# for job in builds[view_and_project]:
# print(" Job: %s" % job)
# build_results = builds[view_and_project][job]
# for build in builds[view_and_project][job]:
# print("%s%s" % (" " * 8, build))
# print("")
#
print("")
print("Elapsed processing time: %6.3f seconds" % (finished - started))
def test_test_konfabulator():
logger = ActivityLogger('test.log')
filename = "config/buildorama.yml"
test_konfabulator = TestKonfabulator(filename, logger)
job1 = 'slippery slope'
item1 = {'Job': job1}
test_konfabulator.add_to_container(item1)
assert test_konfabulator.has_item('Job', job1) == True
job2 = 'uphill both ways'
item2 = {'Job': job2}
test_konfabulator.replace_in_container(item1, item2)
assert test_konfabulator.has_item('Job', job1) == False
assert test_konfabulator.has_item('Job', job2) == True
def main(args):
if args:
config_file_name = args.pop(0)
with open("config/%s" % config_file_name, 'r') as f:
uber_conf = yaml.load(f)
conf = uber_conf['JenkinsBuildConnector']
else:
conf = yaml.load(CONFIG_CHUNK)
ac_conf = conf['AgileCentral']
#pprint(ac_conf)
#print("=" * 80)
#print("")
logger = ActivityLogger("agicen.builds.log", policy='calls', limit=1000)
agicen = AgileCentralConnection(ac_conf, logger)
agicen_headers = {'name' : 'AgileCentral Build Lister', 'version' : __version__,
'vendor' : 'Open Source Galaxy', 'other_version' : 'placeholder' }
agicen.set_integration_header(agicen_headers)
agicen.setSourceIdentification('placeholder', '0.0.0')
agicen.connect()
ref_timestamp = (2016, 3, 1, 0, 0, 0, 5, 0, -1)
started = time.time()
builds = agicen.getRecentBuilds(ref_timestamp)
print("%d Projects had Build items within the defined scope" % len(builds))
finished = time.time()
for project in builds:
##
# if project != ac_conf['DefaultProject']:
# break
##
print("AgileCentral Project: %s" % project)
job_names = builds[project].keys()
for job_name in job_names:
for build in builds[project][job_name]:
build_date = build.Start[:-5].replace('T', ' ')
duration = pretty_duration(build.Duration)
print(" %-24.24s %5d %-10.10s %s %15.15s" % \
(job_name, int(build.Number), build.Status, build_date, duration))
print("")
print("Elapsed retrieval time: %6.3f seconds" % (finished - started))
self.name = name
self.url = url
self.jobs = jobs
def __str__(self):
sub_jobs = len(self.jobs)
return "name: %-24.24s sub-jobs: %3d url: %s " % \
(self.name, sub_jobs, self.url)
def info(self):
return str(self)
############################################################################################
logger = ActivityLogger("jenk.log", policy='calls', limit=1000)
jenk = DepthCharge("tiema03-u183073.ca.com", 8080, 'jenkins', 'rallydev',
logger)
started = time.time()
jenk.connect()
retrieved = time.time()
# for job in jenk.jenk['jobs']:
# if not job['_class'].endswith('.Folder'):
# inventory['jobs'].append(JenkinsJob(job))
# print('---------------------------')
#
# for view in jenk.jenk['views']:
# if not job['_class'].endswith('.ListView'):
class BuildConnectorRunner(object):
def __init__(self, args):
"""
An instance of this class is responsible for basic command line parsing,
pulling off the name of the target config file.
Once that item is taken care of, the instance obtains an instance of a
Konfabulator, a AuxiliaryClassLoader instance and and instance of an BLDConnector.
The Konfabulator is delegated the task of reading the config file, the AuxiliaryClassLoader
is delegated the task of pulling in any Extension classes.
These instances are then provided to the BuildConnector instance.
This instance then directs the BuildConnector instance to obtain unrecorded builds
from the target Build/CI system and reflect them in the Agile Central server.
"""
if len(args) < 1:
problem = (
"Insufficient command line args, must be at least a config file name."
)
raise ConfigurationError(problem)
self.default_log_file_name = True
# this only allows a logfile spec of --logfile=<some_file_name>
# TODO: consider allowing additional syntax logfile=foo.log or --logfile foo.log
log_spec = [arg for arg in args if arg.startswith('--logfile=')]
if log_spec:
self.logfile_name = log_spec[0].replace('--logfile=', '')
self.default_log_file_name = False
for spec in log_spec:
args = [arg for arg in args if arg != spec]
self.config_file_names = args
if self.default_log_file_name: # set it to first config file minus any '_config.yml' portion
self.first_config = self.config_file_names[0]
self.logfile_name = "log/%s.log" % self.first_config.replace(
'.yml', '').replace('_config', '')
try:
if not os.path.exists('log'):
os.makedirs('log')
except Exception as msg:
sys.stderr.write(
"Unable to locate or create the log sub-directory, %s\n" %
msg)
raise Exception
self.log = ActivityLogger(self.logfile_name)
logAllExceptions(True, self.log)
self.preview = False
self.connector = None
self.extension = {}
def proclaim_existence(self, build_system_name):
proc = ProcTable.targetProcess(os.getpid())
now = time.strftime(STD_TS_FMT, time.gmtime(time.time()))
cmd_elements = proc.cmdline.split()
executable = cmd_elements[0]
cmd_elements[0] = os.path.basename(executable)
proc.cmdline = " ".join(cmd_elements)
self.log.write(EXISTENCE_PROCLAMATION %
(build_system_name, __version__, now, proc.pid,
os.getcwd(), proc.cmdline))
def acquireLock(self):
"""
Check for conditions to proceed based on lock file absence/presence status.
Acquisition of the lock is a green-light to proceed.
Failure to acquire the lock prevents any further operation.
"""
if LockFile.exists(LOCK_FILE):
self.log.warning("A %s file exists" % LOCK_FILE)
locker = LockFile.currentLockHolder(LOCK_FILE)
if not LockFile.lockerIsRunning(LOCK_FILE, locker):
message = (
"A prior connector process (%s) did not clean up "
"the lock file on termination, proceeding with this run")
self.log.warning(message % locker)
else:
self.log.error(
"Another connector process [%s] is still running, unable to proceed"
% locker)
raise ConfigurationError(
"Simultaneous processes for this connector are prohibited")
LockFile.createLock(LOCK_FILE)
return True
def releaseLock(self):
LockFile.destroyLock(LOCK_FILE)
def run(self):
own_lock = False
build_system_name = self.identifyBuildSystemName()
self.proclaim_existence(build_system_name)
own_lock = self.acquireLock()
try:
for config_file in self.config_file_names:
config_file_path = self.find_config_file(config_file)
if not config_file_path:
raise ConfigurationError(
"No config file for '%s' found in the config subdir" %
config_file)
lf_name = "log/%s.log" % config_file.replace(
'.yml', '').replace('_config', '')
self.log = ActivityLogger(lf_name)
logAllExceptions(True, self.log)
self._operateService(config_file_path)
except Exception as msg:
self.log.error(msg)
finally:
try:
if own_lock: self.releaseLock()
except Exception as msg:
raise OperationalError(
"ERROR: unable to remove lock file '%s', %s" %
(LOCK_FILE, msg))
self.log.info('run completed')
def identifyBuildSystemName(self):
file_name = self.find_config_file(self.first_config)
if not file_name:
problem = "Unable to locate any config file for the name: '%s'" % self.first_config
sys.stderr.write("ERROR: %s. Correct this and run again.\n" %
problem)
sys.exit(1)
bsn = 'UNKNOWN'
limit = 5
ix = 0
with open(file_name, 'r', encoding="utf-8") as fcf:
while ix < limit:
text_line = fcf.readline()
ix += 1
mo = re.match(r'^(?P<bsn>[A-Z][A-Za-z]+)BuildConnector:$',
text_line)
if mo:
bsn = mo.group('bsn')
break
if bsn == 'UNKNOWN':
problem = "The config file for '%s' does not contain a valid Build Connector identifier." % self.first_config
sys.stderr.write("ERROR: %s. Correct this and run again.\n" %
problem)
sys.exit(2)
return bsn
def find_config_file(self, config_name):
relative_path = 'config/%s' % config_name
if os.path.exists(relative_path):
return relative_path
else:
return None
# valid_targets = ['%s' % config_name, '%s.yml' % config_name]
# hits = [entry for entry in glob.glob('config/*') if config_name in entry]
# if hits:
# return hits[0]
# else:
# return None
def _operateService(self, config_file_path):
config_name = config_file_path.replace('config/', '')
started = finished = elapsed = None
self.connector = None
self.log.info("processing to commence using content from %s" %
config_file_path)
last_conf_mod = time.strftime(
STD_TS_FMT, time.gmtime(os.path.getmtime(config_file_path)))
conf_file_size = os.path.getsize(config_file_path)
self.log.info("%s last modified %s, size: %d chars" %
(config_file_path, last_conf_mod, conf_file_size))
config = self.getConfiguration(config_file_path)
this_run = time.time(
) # be optimistic that the reflectBuildsInAgileCentral service will succeed
now_zulu = time.strftime(STD_TS_FMT, time.gmtime(
this_run)) # zulu <-- universal coordinated time <-- UTC
self.time_file = TimeFile(self.buildTimeFileName(config_name),
self.log)
if self.time_file.exists():
last_run = self.time_file.read(
) # the last_run is in Zulu time (UTC) as an epoch seconds value
else:
last_run = time.time() - (THREE_DAYS)
last_run_zulu = time.strftime(STD_TS_FMT, time.gmtime(last_run))
#self.log.info("Last Run %s --- Now %s" % (last_run_zulu, now_zulu))
self.log.info("Time File value %s --- Now %s" %
(last_run_zulu, now_zulu))
self.connector = BLDConnector(config, self.log)
self.log.debug(
"Got a BLDConnector instance, calling the BLDConnector.run ...")
status, builds = self.connector.run(last_run, self.extension)
# builds is an OrderedDict instance, keyed by job name, value is a list of Build instances
finished = time.time()
elapsed = int(round(finished - this_run))
self.logServiceStatistics(config_name, builds, elapsed)
if self.preview:
self.log.info(
"Preview mode in effect, time.file File not written/updated")
return
if not status and builds:
# Not writing the time.file may cause repetitive detection of Builds,
# but that is better than missing out on Builds altogether
self.log.info(
"There was an error in processing so the time.file was not written"
)
return
if not builds:
self.log.info(
'No builds were added during this run, so the time.file NOT updated.'
)
return
# we've added builds successfully, so update the Time File (config/<config>_time.file)
try:
earliest_build_start = min(
build_list[-1].Start
for job_name, build_list in builds.items())
time_file_timestamp = earliest_build_start.replace('T', ' ')[:19]
self.time_file.write(time_file_timestamp)
self.log.info("time file written with value of %s Z" %
time_file_timestamp)
except Exception as msg:
raise OperationalError(msg)
def buildTimeFileName(self, config_file):
if config_file:
if config_file.endswith('.yml') or config_file.endswith('.cfg'):
time_file_name = config_file[0:-4] + '_time.file'
else:
time_file_name = "%s_time.file" % config_file
else:
time_file_name = 'time.file'
time_file_path = 'log/%s' % time_file_name
return time_file_path
def logServiceStatistics(self, config_name, builds, elapsed):
"""
what we intend to append to the log...
bld_conn_config.yml: 32 additional builds reflected in AgileCentral
and a line with the elapsed time taken in human readable form
(elapsed is in seconds (a float value))
"""
for build in builds:
preview_reminder = "(Preview Mode)" if self.preview else ""
factoid = "%3d builds posted for job %s" % (len(
builds[build]), build)
self.log.info("%s: %s %s" %
(config_name, factoid, preview_reminder))
hours, rem = divmod(elapsed, 3600)
mins, secs = divmod(rem, 60)
if hours > 0:
duration = "%s hours %s minutes %s seconds" % (hours, mins, secs)
else:
if mins > 0:
duration = "%s minutes %s seconds" % (mins, secs)
else:
duration = "%s seconds" % secs
self.log.info("%s: service run took %s" % (config_name, duration))
def getConfiguration(self, config_file):
try:
config = Konfabulator(config_file, self.log)
except NonFatalConfigurationError as msg:
pass # info for this will have already been logged or blurted
except Exception as msg:
raise ConfigurationError(msg)
svc_conf = config.topLevel('Service')
self.preview = False
if svc_conf and svc_conf.get('Preview', None) == True:
self.preview = True
self.log_level = 'Info'
if svc_conf:
ll = svc_conf.get('LogLevel', 'Info').title()
if ll in ['Fatal', 'Error', 'Warn', 'Info', 'Debug']:
self.log_level = ll
self.log.setLevel(self.log_level)
else:
pass # bad LogLevel specified
#if 'PostBatchExtension' in svc_conf:
# pba_class_name = svc_conf['PostBatchExtension']
# pba_class = ExtensionLoader().getExtension(pba_class_name)
# self.extension['PostBatch'] = pba_class()
return config