def main_create_jobs(self):
c = self.console
ib_file = find_infrabox_file(self.mount_repo_dir)
if not ib_file:
raise Failure("infrabox file not found")
c.header("Parsing infrabox file", show=True)
data = self.parse_infrabox_file(ib_file)
self.check_file_exist(data, self.mount_repo_dir)
c.header("Creating jobs", show=True)
jobs = self.get_job_list(data, c, self.job['repo'], infrabox_paths={ib_file: True})
if jobs:
self.create_jobs(jobs)
c.collect("Done creating jobs")
else:
c.collect("No jobs")
def create_dynamic_jobs(self):
c = self.console
ib_file = find_infrabox_file(self.infrabox_output_dir)
if ib_file:
infrabox_context = os.path.dirname(ib_file)
c.header("Creating jobs", show=True)
data = self.parse_infrabox_file(ib_file)
self.check_file_exist(data, infrabox_context)
jobs = self.get_job_list(data, c, self.job['repo'],
infrabox_paths={ib_file: True},
infrabox_context=infrabox_context)
c.collect(json.dumps(jobs, indent=4), show=True)
if jobs:
self.create_jobs(jobs)
c.collect("Done creating jobs")
else:
c.collect("No jobs")
def get_job_list(self, data, c, repo, parent_name="",
infrabox_context=None, infrabox_paths=None):
#pylint: disable=too-many-locals
if not infrabox_context:
infrabox_context = self.mount_repo_dir
if not infrabox_paths:
infrabox_paths = {}
self.rewrite_depends_on(data)
jobs = []
for job in data['jobs']:
job['id'] = str(uuid.uuid4())
job['avg_duration'] = 0
job['repo'] = repo
job['infrabox_context'] = os.path.normpath(infrabox_context)
if parent_name != '':
job['name'] = parent_name + "/" + job['name']
deps = job.get('depends_on', [])
for x in xrange(0, len(deps)):
deps[x]['job'] = parent_name + "/" + deps[x]['job']
job_name = job['name']
if job['type'] != "workflow" and job['type'] != "git":
jobs.append(job)
continue
if job['type'] == "git":
c.header("Clone repo %s" % job['clone_url'], show=True)
clone_url = job['clone_url']
branch = job.get("branch", None)
sub_path = os.path.join('.infrabox', 'tmp', job_name)
new_repo_path = os.path.join(self.mount_repo_dir, sub_path)
c.execute(['rm', '-rf', new_repo_path])
os.makedirs(new_repo_path)
self.clone_repo(job['commit'], clone_url, branch, None, True, sub_path)
c.header("Parsing infrabox file", show=True)
ib_file = job.get('infrabox_file', None)
if not ib_file:
ib_path = find_infrabox_file(new_repo_path)
else:
ib_path = os.path.join(new_repo_path, ib_file)
if not ib_path:
raise Failure("infrabox file not found in %s" % new_repo_path)
if ib_path in infrabox_paths:
raise Failure("Recursive include detected")
infrabox_paths[ib_path] = True
c.collect("file: %s" % ib_path)
yml = self.parse_infrabox_file(ib_path)
git_repo = {
"clone_url": job['clone_url'],
"commit": job['commit'],
"infrabox_file": ib_file,
"clone_all": True,
"branch": job.get('branch', None)
}
new_infrabox_context = os.path.dirname(ib_path)
self.check_file_exist(yml, new_infrabox_context)
sub = self.get_job_list(yml, c, git_repo,
parent_name=job_name,
infrabox_context=new_infrabox_context,
infrabox_paths=infrabox_paths)
for s in sub:
s['infrabox_context'] = s['infrabox_context'].replace(new_repo_path, self.mount_repo_dir)
del infrabox_paths[ib_path]
if job['type'] == 'workflow':
c.header("Parsing infrabox file", show=True)
p = os.path.join(infrabox_context, job['infrabox_file'])
c.collect("file: %s" % p)
if p in infrabox_paths:
raise Failure("Recursive include detected")
infrabox_paths[p] = True
yml = self.parse_infrabox_file(p)
new_infrabox_context = os.path.dirname(p)
self.check_file_exist(yml, new_infrabox_context)
sub = self.get_job_list(yml, c, repo,
parent_name=job_name,
infrabox_context=new_infrabox_context)
del infrabox_paths[p]
# every sub job which does not have a parent
# should be a child of the current job
job_with_children = {}
for s in sub:
deps = s.get('depends_on', [])
if not deps:
s['depends_on'] = job.get('depends_on', [])
for d in deps:
job_with_children[d['job']] = True
# overwrite env vars if set
if 'environment' in job:
for n, v in job['environment'].iteritems():
if 'environment' not in s:
s['environment'] = {}
s['environment'][n] = v
jobs += sub
# add a wait job to all sub jobs
# which don't have a child, so we have
# one 'final' job
final_job = {
"type": "wait",
"name": job_name,
"depends_on": [],
"id": str(uuid.uuid4())
}
for s in sub:
sub_name = s['name']
if sub_name not in job_with_children:
final_job['depends_on'].append({"job": sub_name, "on": ["finished"]})
jobs.append(final_job)
return jobs