def fill_blocks(self, blk_lst):
"""to fill blocks config dir after initialization"""
env_boot.EnvBoot.__init__(self)
self.boot_env()
for blk_name in blk_lst:
LOG.info(":: filling block %s ...", blk_name)
os.environ["BLK_NAME"] = blk_name
os.environ[
"BLK_ROOT"] = blk_root_dir = f"{self.ced['PROJ_ROOT']}{os.sep}{blk_name}"
pcom.mkdir(LOG, blk_root_dir)
proj_cfg_dir = os.path.expandvars(settings.PROJ_CFG_DIR).rstrip(
os.sep)
blk_cfg_dir = os.path.expandvars(settings.BLK_CFG_DIR).rstrip(
os.sep)
for cfg_kw in self.cfg_dic:
if cfg_kw in settings.BLK_CFG_UNFILL_LST:
continue
proj_cfg = f"{proj_cfg_dir}{os.sep}{cfg_kw}.cfg"
blk_cfg = f"{blk_cfg_dir}{os.sep}{cfg_kw}.cfg"
LOG.info("generating block config %s", blk_cfg)
pcom.mkdir(LOG, os.path.dirname(blk_cfg))
with open(proj_cfg) as pcf, open(blk_cfg, "w") as bcf:
for line in pcom.gen_pcf_lst(pcf):
bcf.write(line)
for dir_cfg_kw in self.dir_cfg_dic:
if dir_cfg_kw == "lib":
continue
proj_dir_cfg = f"{proj_cfg_dir}{os.sep}{dir_cfg_kw}"
blk_dir_cfg = f"{blk_cfg_dir}{os.sep}{dir_cfg_kw}{os.sep}DEFAULT"
LOG.info("generating block config directory %s", blk_dir_cfg)
if os.path.isdir(blk_dir_cfg):
LOG.info(
"block level config directory %s already exists, "
"please confirm to overwrite it", blk_dir_cfg)
pcom.cfm()
shutil.rmtree(blk_dir_cfg, True)
shutil.copytree(proj_dir_cfg, blk_dir_cfg)
for blk_cfg in pcom.find_iter(blk_dir_cfg,
"*.cfg",
cur_flg=True):
with open(blk_cfg) as ocf:
blk_lines = pcom.gen_pcf_lst(ocf)
with open(blk_cfg, "w") as ncf:
for line in blk_lines:
ncf.write(line)
proj_share_dir = os.path.expandvars(settings.PROJ_SHARE).rstrip(
os.sep)
proj_blk_cmn_dir = f"{proj_share_dir}{os.sep}block_common"
blk_cmn_dir = f"{blk_root_dir}{os.sep}block_common"
if not os.path.isdir(proj_blk_cmn_dir):
continue
if os.path.isdir(blk_cmn_dir):
LOG.info(
"block level common directory %s already exists, "
"please confirm to overwrite it", blk_cmn_dir)
pcom.cfm()
shutil.rmtree(blk_cmn_dir, True)
shutil.copytree(proj_blk_cmn_dir, blk_cmn_dir)
def list_diff(self, flow_name='DEFAULT'):
"""toggle to demonstrate the diff between block level config/plugins
and proj level config/plugins"""
if not self.blk_flg:
LOG.error("it's not in a block directory, please cd into one")
raise SystemExit()
LOG.info(":: diff list for the flows ...")
proj_cfg_dir = self.ced["PROJ_SHARE_CFG"].rstrip(os.sep)
blk_cfg_dir = self.ced["BLK_CFG"].rstrip(os.sep)
LOG.info('proj config dir: %s', proj_cfg_dir)
LOG.info('block config dir: %s', blk_cfg_dir)
# diff listing for config files
for cfg_kw in self.cfg_dic:
if cfg_kw in settings.BLK_CFG_UNFILL_LST:
continue
proj_cfg = f"{proj_cfg_dir}{os.sep}{cfg_kw}.cfg"
blk_cfg = f"{blk_cfg_dir}{os.sep}{cfg_kw}.cfg"
if not os.path.isfile(blk_cfg):
LOG.warning("only in proj config dir: %s", proj_cfg)
continue
with open(proj_cfg) as pcf, open(blk_cfg) as bcf:
proj_cfg_lines = pcom.gen_pcf_lst(pcf)
blk_cfg_lines = bcf.readlines()
LOG.info('diff %s ...', blk_cfg)
os.sys.stdout.writelines(
unified_diff(proj_cfg_lines, blk_cfg_lines, fromfile=proj_cfg, tofile=blk_cfg))
# diff listing for config dirs
for dir_cfg_kw in self.dir_cfg_dic:
if dir_cfg_kw == "lib":
continue
proj_dir_cfg = f"{proj_cfg_dir}{os.sep}{dir_cfg_kw}"
blk_dir_cfg = f"{blk_cfg_dir}{os.sep}{dir_cfg_kw}{os.sep}{flow_name}"
LOG.info("diff block config dir. %s", blk_dir_cfg)
internal_diff_tool(blk_dir_cfg, proj_dir_cfg)
def merge_files(self, dcmp):
""" launch diff for different files """
proj_cfg_str = f"{os.sep}share{os.sep}"
for right_only in dcmp.right_only:
if right_only.startswith("."):
continue
LOG.info("only in destination dir (%s) : %s", dcmp.right,
right_only)
for left_only in dcmp.left_only:
if left_only.startswith("."):
continue
LOG.info("only in source dir (%s) : %s", dcmp.left, left_only)
choice = input("Sync to destination dir? [y/N]: ").lower()
src = f"{dcmp.left}{os.sep}{left_only}"
dst = f"{dcmp.right}{os.sep}{left_only}"
if choice and choice[0] == 'y':
if os.path.isdir(src):
shutil.copytree(src, dst)
elif os.path.isfile(src):
pcom.mkdir(LOG, os.path.dirname(dst))
if proj_cfg_str in dcmp.left and proj_cfg_str not in dcmp.right:
with open(src) as src_f, open(dst, "w") as dst_f:
for line in pcom.gen_pcf_lst(src_f):
dst_f.write(line)
else:
shutil.copyfile(src, dst)
for name in dcmp.diff_files:
if name.startswith("."):
continue
LOG.info("different file %s found in %s", name, dcmp.left)
src = f"{dcmp.left}{os.sep}{name}"
dst = f"{dcmp.right}{os.sep}{name}"
if proj_cfg_str in dcmp.left and proj_cfg_str not in dcmp.right:
ntf = tempfile.NamedTemporaryFile("w")
with open(src) as src_f:
for line in pcom.gen_pcf_lst(src_f):
ntf.write(line)
ntf.seek(0)
if not cmp(ntf.name, dst):
diff_str = f"{self.tool} {ntf.name} {dst}"
subprocess.run(diff_str, shell=True)
ntf.close()
else:
diff_str = f"{self.tool} {src} {dst}"
subprocess.run(diff_str, shell=True)
for sub_dcmp in dcmp.subdirs.values():
self.merge_files(sub_dcmp)
def merge_file(self):
""" launch diff for different file """
proj_cfg_str = f"{os.sep}share{os.sep}"
src = self.src_dir
dst = self.dst_dir
if proj_cfg_str in self.src_dir:
ntf = tempfile.NamedTemporaryFile("w")
with open(src) as src_f:
for line in pcom.gen_pcf_lst(src_f):
ntf.write(line)
ntf.seek(0)
if not cmp(ntf.name, dst):
diff_str = f"{self.tool} {ntf.name} {dst}"
subprocess.run(diff_str, shell=True)
ntf.close()
else:
diff_str = f"{self.tool} {src} {dst}"
subprocess.run(diff_str, shell=True)
def diff_files(dcmp):
"""listing file diff"""
for left_only in dcmp.left_only:
LOG.info("only in block config dir.: %s%s%s", dcmp.left, os.sep, left_only)
for right_only in dcmp.right_only:
LOG.info("only in proj config dir. : %s%s%s", dcmp.right, os.sep, right_only)
for name in dcmp.diff_files:
proj_file = f"{dcmp.right}{os.sep}{name}"
blk_file = f"{dcmp.left}{os.sep}{name}"
LOG.info("diff %s ...", blk_file)
with open(proj_file) as pff, open(blk_file) as bff:
if fnmatch(name, '*.cfg'):
proj_file_lines = pcom.gen_pcf_lst(pff)
else:
proj_file_lines = pff.readlines()
blk_file_lines = bff.readlines()
os.sys.stdout.writelines(
unified_diff(proj_file_lines, blk_file_lines, fromfile=proj_file, tofile=blk_file))
for sub_dcmp in dcmp.subdirs.values():
diff_files(sub_dcmp)
def init(self, init_lst):
"""to perform flow initialization"""
if not self.blk_flg:
LOG.error("it's not in a block directory, please cd into one")
raise SystemExit()
for init_name in init_lst:
LOG.info(":: initializing flow %s directories ...", init_name)
parent_flow = pcom.rd_cfg(self.cfg_dic.get("flow", {}), init_name, "pre_flow", True)
if parent_flow:
src_dir = f"{self.ced['BLK_CFG_FLOW']}{os.sep}{parent_flow}"
LOG.info("inheriting from %s", parent_flow)
else:
src_dir = f"{self.ced['PROJ_SHARE_CFG']}{os.sep}flow"
LOG.info("inheriting from project share")
dst_dir = f"{self.ced['BLK_CFG_FLOW']}{os.sep}{init_name}"
if not os.path.isdir(src_dir):
LOG.error("parent flow directory %s is NA", src_dir)
raise SystemExit()
if os.path.isdir(dst_dir):
if self.cfm_yes:
LOG.warning(
"initializing flow directory %s already exists, "
"confirmed to overwrite the previous flow config and plugins", dst_dir)
else:
LOG.info(
"initializing flow directory %s already exists, "
"please confirm to overwrite the previous flow config and plugins", dst_dir)
pcom.cfm()
shutil.rmtree(dst_dir, True)
shutil.copytree(src_dir, dst_dir)
if not parent_flow:
for blk_cfg in pcom.find_iter(dst_dir, "*.cfg", cur_flg=True):
with open(blk_cfg) as ocf:
blk_lines = pcom.gen_pcf_lst(ocf)
with open(blk_cfg, "w") as ncf:
for line in blk_lines:
ncf.write(line)