def __init__(self, cfgfile):
with open(cfgfile, "r") as fd:
self.__lconfig = json.load(fd)
self.__name = self.__lconfig['name']
self.__tmp_dir = self.__lconfig['global_tmp_dir']
self.__artifacts_dir = os.path.join(self.__tmp_dir, "artifacts")
os.makedirs(self.__artifacts_dir, exist_ok=True)
self.__working_dir = os.path.join(self.__tmp_dir, ".mincid")
with open(
os.path.join(self.__working_dir,
'project_config.json'), "r") as fd:
self.__global_config = json.load(fd)
with open(os.path.join(self.__working_dir,
"mincid_master.json"), "r") as fd:
self.__master_config = json.load(fd)
self.__config = Config(self.__working_dir,
self.__lconfig['branch_name'])
self.__name = self.__lconfig['name']
self.__variant_dir = self.__lconfig['directory']
self.__logger = MLogger("Variant", self.__name, self.__variant_dir)
self.__logger.info("Init variant [%s]" % self.__lconfig['name'])
self.__cmds = []
self.__build_pre_cmds = [r'true']
self.__cmds_post = [r'true']
def __init__(self, branch_name, tmp_dir):
self.__name = branch_name
self.__tmp_dir = tmp_dir
self.__working_dir = os.path.join(self.__tmp_dir, ".mincid")
with open(os.path.join(self.__working_dir, "mincid_master.json"), "r") as fd:
self.__master_config = json.load(fd)
self.__jobids = {}
self.__config = Config(self.__working_dir, branch_name)
self.__branch_dir = os.path.join(self.__working_dir,
branch_name.replace("/", "_"))
self.__logger = MLogger("Branch", "build_branch", self.__branch_dir)
self.__logger.info("Init branch [%s]" % (self.__name))
self.__variants_base_dir = os.path.join(self.__branch_dir, "variants")
os.makedirs(self.__variants_base_dir)
class Branch(object):
def __init__(self, branch_name, tmp_dir):
self.__name = branch_name
self.__tmp_dir = tmp_dir
self.__working_dir = os.path.join(self.__tmp_dir, ".mincid")
with open(os.path.join(self.__working_dir, "mincid_master.json"), "r") as fd:
self.__master_config = json.load(fd)
self.__jobids = {}
self.__config = Config(self.__working_dir, branch_name)
self.__branch_dir = os.path.join(self.__working_dir,
branch_name.replace("/", "_"))
self.__logger = MLogger("Branch", "build_branch", self.__branch_dir)
self.__logger.info("Init branch [%s]" % (self.__name))
self.__variants_base_dir = os.path.join(self.__branch_dir, "variants")
os.makedirs(self.__variants_base_dir)
def __sort_deps(self):
"""Sorts the branch jobs using the dependencies"""
jc = self.__config.branch_jobs()
result = []
todo = set(jc.keys())
while len(todo) > 0:
for k in todo:
if not 'depends_on' in jc[k]:
# No dependecies: can start right away
result.append(k)
todo.remove(k)
break
else:
don = jc[k]['depends_on']
if set(don).issubset(set(result)):
# Yep: all dependencies are already handled!
result.append(k)
todo.remove(k)
break
return result
def __start_variant(self, sname, image, variant_list):
base = self.__config.expand(image['base'])
self.__logger.info("Start variant [%s] [%s] [%s]"
% (sname, base, variant_list))
stdouterr_filename = os.path.join(self.__branch_dir,
"start_variants.stdouterr")
dep_jobids = []
# Collect jobids of all dependend stages
if 'depends_on' in self.__config.branch_jobs()[sname]:
for dep_stage in self.__config.branch_jobs()[sname]['depends_on']:
dep_jobids.extend(self.__jobids[dep_stage])
self.__logger.info("Dependent jobids [%s]" % dep_jobids)
with open(stdouterr_filename, "w") as fd_stdouterr:
# Create own temp dir
rfsname = RFSString(self.__name)
rfsbase = RFSString(base)
variant_name = "%s+%s+%s+%s+%s" % \
(self.__config.project_cfg('name'),
rfsname.fs(),
sname, rfsbase.fs(), "_".join(variant_list))
variant_tmp_dir = os.path.join(self.__variants_base_dir, variant_name)
os.makedirs(variant_tmp_dir, exist_ok=True)
variant_desc = {
'name': variant_name,
'base': base,
'branch_name': self.__name,
'global_tmp_dir': self.__tmp_dir,
'directory': variant_tmp_dir
}
if len(variant_list)>0:
variant_desc['variant_list'] = variant_list
for pword in ('install', 'run'):
if pword in self.__config.branch_jobs()[sname]:
variant_desc[
pword] = self.__config.branch_jobs()[sname][pword]
if 'prepare' in image:
variant_desc['prepare'] = image['prepare']
if 'build_prepare' in image:
variant_desc['build_prepare'] = image['build_prepare']
variant_cfg_file_name = os.path.join(
variant_tmp_dir, "variant.json")
with open(variant_cfg_file_name, "w") as fd:
json.dump(variant_desc, fd)
subproc_slist = [
"sbatch", "--job-name=%s" % variant_name,
"--output=%s" % os.path.join(self.__working_dir,
"slurm_build_branch_%j.out"),
"--extra-node-info=1:2:1",
"--export=PYTHONPATH"]
if len(dep_jobids)>0:
dep_str = ":".join(str(x) for x in dep_jobids)
self.__logger.info("New batch is dependent on [%s]" %
dep_str)
subproc_slist.append("--dependency=afterok:%s" % dep_str)
subproc_slist.extend(
[os.path.join(self.__master_config['mincid_install_dir'],
"build_variant.py"), variant_cfg_file_name])
p = subprocess.Popen(subproc_slist,
stdout=subprocess.PIPE, stderr=fd_stdouterr)
res = p.stdout.read()
p.wait()
self.__logger.info("sbatch process return value [%d]" % p.returncode)
decoded = res.decode("UTF-8")
jobnr = int(decoded[20:-1])
self.__jobids[sname].append(jobnr)
self.__logger.info("sbatch process id [%d]" % jobnr)
self.__logger.info("Finished variant [%s] [%s] [%s]"
% (sname, base, variant_list))
def __start_variants(self, sname, image):
self.__logger.info("Start variants [%s] [%s]" % (sname, image))
variants = list(itertools.product(*image['variants']))
for variant_list in variants:
self.__start_variant(sname, image, variant_list)
self.__logger.info("Finished variants [%s] [%s]" % (sname, image))
def __start_image(self, sname, image):
jc = self.__config.branch_jobs()
base = self.__config.expand(image['base'])
if 'ignore' in image and image['ignore']=="true":
self.__logger.info("Ignoring image [%s] [%s]" % (sname, base))
return
self.__logger.info("Start image [%s] [%s]" % (sname, base))
if 'variants' in image:
self.__start_variants(sname, image)
else:
self.__start_variant(sname, image, [])
self.__logger.info("Finished image [%s] [%s]" % (sname, base))
def __start_stage(self, sname):
self.__logger.info("Start stage [%s]" % sname)
for image in self.__config.branch_jobs()[sname]['images']:
self.__start_image(sname, image)
self.__logger.info("Finished stage [%s]" % sname)
def __start_stages(self, nlist):
self.__logger.info("Start all stages [%s]" % (nlist))
# Store all jobids for the appropriate (high level) job
for n in nlist:
self.__jobids[n] = []
for n in nlist:
self.__start_stage(n)
self.__logger.info("Finished stating all stages [%s]" % (nlist))
def process(self):
self.__logger.info("Process branch [%s]" % (self.__name))
nlist = self.__sort_deps()
self.__logger.info("Node list [%s]" % (nlist))
self.__start_stages(nlist)
class Variant(object):
def __init__(self, cfgfile):
with open(cfgfile, "r") as fd:
self.__lconfig = json.load(fd)
self.__name = self.__lconfig['name']
self.__tmp_dir = self.__lconfig['global_tmp_dir']
self.__artifacts_dir = os.path.join(self.__tmp_dir, "artifacts")
os.makedirs(self.__artifacts_dir, exist_ok=True)
self.__working_dir = os.path.join(self.__tmp_dir, ".mincid")
with open(
os.path.join(self.__working_dir,
'project_config.json'), "r") as fd:
self.__global_config = json.load(fd)
with open(os.path.join(self.__working_dir,
"mincid_master.json"), "r") as fd:
self.__master_config = json.load(fd)
self.__config = Config(self.__working_dir,
self.__lconfig['branch_name'])
self.__name = self.__lconfig['name']
self.__variant_dir = self.__lconfig['directory']
self.__logger = MLogger("Variant", self.__name, self.__variant_dir)
self.__logger.info("Init variant [%s]" % self.__lconfig['name'])
self.__cmds = []
self.__build_pre_cmds = [r'true']
self.__cmds_post = [r'true']
def __variant_cmds(self, variant_flat):
# Check if re matches
for vcfg in self.__master_config['imagedef'][self.__lconfig['base']]['variant_cmds']:
vre = vcfg[0]
vcmd = vcfg[1]
self.__logger.debug("RE search [%s] [%s]" % (vre, variant_flat))
result = re.search(vre, variant_flat)
if not result:
continue
for icmd in vcmd:
self.__logger.info("Calling Sub [%s] [%s] [%s]" %
(vre, icmd, variant_flat))
nsub = re.sub(vre, icmd, variant_flat)
self.__logger.info("Add cmd [%s]" % nsub)
self.__cmds.append(nsub)
if len(vcfg)>2:
vpreb = vcfg[2]
for ipreb in vpreb:
self.__logger.info("Calling Sub [%s] [%s] [%s]"
% (vre, ipreb, variant_flat))
nsub = re.sub(vre, ipreb, variant_flat)
self.__logger.info("Add pre cmd [%s]" % nsub)
self.__build_pre_cmds.append(nsub)
def __variant_install_pkgs(self, install_dict):
# Add other packages...
pkgs=[]
if not 'pkgs' in install_dict:
return
rpkgs = self.__config.expand(install_dict['pkgs'])
install_pkg_names = self.__master_config['imagedef'][self.__lconfig['base']]['pkg_names']
for ipkg in rpkgs:
if not ipkg in install_pkg_names:
self.__logger.warning(
"Package with symolic name [%s] " % ipkg
+ "has no name mapping")
pkgs.append(ipkg)
else:
pkgs.extend(install_pkg_names[ipkg])
if len(pkgs)>0:
self.__cmds.append(
"%s %s" % (
self.__master_config['imagedef'][self.__lconfig['base']]['install_pkg_cmd'],
(" ".join(pkgs))))
def __variant_cmds_post(self, install_dict):
# Add packages from the control file
# Please note that this can happen AFTER the checkout!
if not 'control_files' in install_dict:
self.__logger.info("No control files given")
return
for cf in install_dict['control_files']:
self.__logger.info("Control file [%s]" % cf)
self.__cmds_post.append(
"cd ~builder && mk-build-deps "
+ "--tool='apt-get -y --no-install-recommends' "
+ "--install %s" % cf)
def __image_prepare(self):
if not 'prepare' in self.__lconfig:
self.__logger.info("No prepare given")
return
self.__cmds.extend(self.__lconfig['prepare'])
def __image_build_prepare(self):
if not 'build_prepare' in self.__lconfig:
self.__logger.info("No build prepare given")
return
self.__cmds_post.extend(self.__lconfig['build_prepare'])
def __handle_variant(self):
variant_flat = ":" + ":".join(self.__lconfig['variant_list']) + ":"
self.__logger.info("Flat variant [%s]" % variant_flat)
self.__variant_cmds(variant_flat)
def __global_build_pre_cmds(self):
if "build_preparation_cmds" in self.__master_config:
self.__build_pre_cmds.extend(self.__master_config["build_preparation_cmds"])
def process(self):
self.__logger.info("Start variant [%s]" % self.__name)
self.__image_prepare()
self.__image_build_prepare()
self.__global_build_pre_cmds()
if 'variant_list' in self.__lconfig:
self.__handle_variant()
if 'install' in self.__lconfig:
self.__variant_install_pkgs(self.__lconfig['install'])
self.__variant_cmds_post(self.__lconfig['install'])
stdouterr_filename = os.path.join(
self.__tmp_dir, self.__name + ".log")
rm_docker_image = "true"
if "remove_docker_image" in self.__master_config['imagedef'][self.__lconfig['base']]:
rm_docker_image = self.__master_config['imagedef'][self.__lconfig['base']]['remove_docker_image']
setup_minimalistic=""
if 'setup_image_minimalistic' in self.__master_config['imagedef'][self.__lconfig['base']]:
setup_minimalistic="\n".join(self.__master_config['imagedef'][self.__lconfig['base']]['setup_image_minimalistic'])
# Log all commands that are executed
complete_docker_cmd = ["docker", "run", "--rm=%s" % rm_docker_image, "-i",
"-v", "%s:/working:rw" % self.__working_dir,
"-v", "%s:/artifacts:rw" % self.__artifacts_dir,
"-v", "%s:/resources:ro" % self.__master_config["resource_dir"],
self.__lconfig['base'],
"/bin/bash", "-x", "-e"]
complete_build_cmds = """%s
%s
%s
su - builder --command "%s"
su - builder --command 'git clone %s --branch %s --single-branch %s'
%s
su - builder --command '%s && %s'
""" % \
(setup_minimalistic,
"\n".join(self.__master_config['imagedef'][self.__lconfig['base']]['setup_image']),
"\n".join(self.__cmds),
self.__global_config['vcs']['authcmd'],
self.__global_config['vcs']['url'],
self.__lconfig['branch_name'],
self.__global_config['dest'],
" && ".join(self.__cmds_post),
"\n".join(self.__build_pre_cmds),
self.__lconfig['run'] if 'run' in self.__lconfig else "")
with open(os.path.join(self.__tmp_dir, self.__name + ".sh"), "w") as shlog:
shlog.write("# %s\n" % " ".join(complete_docker_cmd))
shlog.write(complete_build_cmds)
with open(stdouterr_filename, "w") as fd_stdouterr:
p = subprocess.Popen(complete_docker_cmd,
stdin=subprocess.PIPE,
stdout=fd_stdouterr, stderr=fd_stdouterr)
p.stdin.write(bytes(complete_build_cmds, 'UTF-8'))
p.stdin.close()
p.wait()
self.__logger.info("Docker process return value [%d]" % p.returncode)
self.__logger.info("Finished variant [%s]" % (self.__name))
with open(os.path.join(self.__tmp_dir, "results.txt"), "a") as fd:
fd.write("%s: %s\n" % (self.__name, "success" if p.returncode == 0 else "failure"))
# Exit with the result of the docker (script)
sys.exit(p.returncode)