def build_module(module, required):
import os.path
tools.mkdir_safe('product/modules/%s' % module)
out = 'product/modules/%s/%s.o' % (module, module)
srcs = []
metafile = 'modules/%s/%s.meta' % (module, module)
if os.path.exists(metafile):
tools.copy_file('product/modules/%s/%s.meta' % (module, module), metafile)
srcs.append(metafile)
meta = tools.load_meta(metafile)
extra_objs = ''
if 'objs' in meta:
extra_objs = meta['objs']
src = 'modules/%s/%s.rip' % (module, module)
if tools.depends(out, module_deps + [src]):
tools.pprint('MOD', src, out)
args = ['product/ripe', conf["RFLAGS"], '-n', module,
'-c', srcs, src, extra_objs, '-o', out]
# Required (default) packages have already been typed, and are
# loaded by default. Hence, they do not need to be typed.
if required:
args.append('--omit-typing')
if conf["VERBOSITY"] > 1:
args.append('-v')
if required:
tools.call(args)
else:
if not tools.try_call(args):
failed_modules.append(module)
def create_setenv_script(self):
"""
Create the 'setenv.sh' file from the 'templates/setenv.tpl' template under the
lfsbuilder source directory.
"""
setenv_filename = os.path.join(
self.builder_data_dict["setenv_directory"],
self.builder_data_dict["setenv_filename"])
template = os.path.join(
self.builder_data_dict["lfsbuilder_templates_directory"],
self.builder_data_dict["setenv_template"])
substitution_list = [
"@@LFS_BUILDER_NAME@@", self.builder_data_dict["name"],
"@@LFS_ENV_PATH_VALUE@@", self.builder_data_dict["env_PATH_value"],
"@@LFS_SOURCES_DIRECTORY@@",
self.builder_data_dict["sources_directory"],
"@@LFS_TOOLS_DIRECTROY@@",
self.builder_data_dict["tools_directory"],
"@@LFSBUILDER_SRC_DIRECTORY@@",
self.builder_data_dict["lfsbuilder_src_directory"],
"@@LFS_BASE_DIRECTORY@@", config.BASE_DIRECTORY,
"@@LFS_MAKEFLAGS@@", config.MAKEFLAGS
]
# Copy template
tools.copy_file(template, setenv_filename)
# Substitute parameters
tools.substitute_multiple_in_file(setenv_filename, substitution_list)
def build_module(module, required):
import os.path
tools.mkdir_safe('product/modules/%s' % module)
out = 'product/modules/%s/%s.o' % (module, module)
srcs = []
metafile = 'modules/%s/%s.meta' % (module, module)
if os.path.exists(metafile):
tools.copy_file('product/modules/%s/%s.meta' % (module, module),
metafile)
srcs.append(metafile)
meta = tools.load_meta(metafile)
extra_objs = ''
if 'objs' in meta:
extra_objs = meta['objs']
src = 'modules/%s/%s.rip' % (module, module)
if tools.depends(out, module_deps + [src]):
tools.pprint('MOD', src, out)
args = [
'product/ripe', conf["RFLAGS"], '-n', module, '-c', srcs, src,
extra_objs, '-o', out
]
# Required (default) packages have already been typed, and are
# loaded by default. Hence, they do not need to be typed.
if required:
args.append('--omit-typing')
if conf["VERBOSITY"] > 1:
args.append('-v')
if required:
tools.call(args)
else:
if not tools.try_call(args):
failed_modules.append(module)
def copy_script_header(self, script_filename):
"""
Copy script header from the 'script.tpl' template under the 'templates' directory.
"""
template = os.path.join(self.component_data_dict["lfsbuilder_templates_directory"],
self.component_data_dict["script_template"])
printer.substep_info("Generating script '{f}'".format(f=script_filename))
tools.copy_file(template, script_filename)
def clean_up(self):
'''Move exception files into the exception folder.'''
dont_rename = self.parse_failed + self.non_abi
tools.move_file(dont_rename, self.fail_folder)
for entry in self.exceptions:
src = entry[1]
dst = os.path.join(self.fail_folder, entry[2])
tools.copy_file(src, dst)
os.remove(entry[1])
assert (os.listdir(self.temp_dir), [])
shutil.rmtree(self.temp_dir)
def clean_up(self):
'''Move exception files into the exception folder.'''
dont_rename = self.parse_failed + self.non_abi
tools.move_file(dont_rename, self.fail_folder)
for entry in self.exceptions:
src = entry[1]
dst = os.path.join(self.fail_folder, entry[2])
tools.copy_file(src, dst)
os.remove(entry[1])
assert(os.listdir(self.temp_dir), [])
shutil.rmtree(self.temp_dir)
def run_previous_steps(component_data_dict, parent_function):
# Call parent function
parent_function()
print("Copying custom \'.config\' file")
filename = os.path.join(component_data_dict["lfsbuilder_src_directory"],
"recipes", "components", "kernel", "files",
component_data_dict["kernel_config_filename"])
tools.copy_file(
filename,
os.path.join(component_data_dict["extracted_directory"], ".config"))
def copy_to_organized_directory(dir_src: str, dir_dst: str) -> bool:
"""copy source directory to organized directory
Get content from source directory and copy to a new directory
into new folders with the extentions name
Arguments:
dir_src {str} -- source directory
dir_dst {str} -- destination directory
Raises:
IOError: path dir_src not found
IOError: path dir_dst not found
Returns:
bool -- if successful return True
"""
if not checks.check_exists(dir_src):
raise IOError("Directory '{0}' not found.".format(dir_src))
if not checks.check_exists(dir_dst):
raise IOError("Directory '{0}' not found.".format(dir_dst))
# copy files temp to new directory
list_files = get_files(dir_src)
for file in list_files:
basename = os.path.basename(file)
search = re.search(r"\.([A-Za-z0-9]+)$", basename)
if search:
extension = search.group(1)
# check = [ext for ext in extensions if search.group(1) == ext]
directory_extension = os.path.join(dir_dst, extension)
if not checks.check_exists(directory_extension):
tools.create_directory(directory_extension)
# build target directory
target_directory = os.path.join(directory_extension, basename)
# copy file
tools.copy_file(file, target_directory)
return True
def split_dataset(rate):
if not os.path.exists(train_dir):
os.makedirs(train_dir)
if not os.path.exists(val_dir):
os.makedirs(val_dir)
files = os.listdir(org_dir)
for name in files:
tools.copy_file(org_dir + name, train_dir + name)
file_num = len(files)
pick_num = int(file_num * rate)
print(file_num)
print(pick_num)
sample = random.sample(files, pick_num)
for name in sample:
tools.move_file(train_dir + name, val_dir + name)
return
def kaldi_adapt_lm(kaldi_root, src_model_dir, lm_fn, work_dir, dst_model_name):
steps_path = '%s/egs/wsj/s5/steps' % kaldi_root
if not os.path.exists(steps_path):
raise Exception(
'%s does not exist - is kaldi really installed in %s ?' %
(steps_path, kaldi_root))
tmpl_dir = os.path.dirname(os.path.abspath(__file__)) + '/templates'
#
# copy dictionary and phoneme sets from original model
#
logging.info("copying dictionary and phoneme sets from original model...")
misc.mkdirs('%s/data/local/dict' % work_dir)
misc.copy_file('%s/data/local/dict/lexicon.txt' % src_model_dir,
'%s/data/local/dict/lexicon.txt' % work_dir)
misc.copy_file('%s/data/local/dict/nonsilence_phones.txt' % src_model_dir,
'%s/data/local/dict/nonsilence_phones.txt' % work_dir)
misc.copy_file('%s/data/local/dict/silence_phones.txt' % src_model_dir,
'%s/data/local/dict/silence_phones.txt' % work_dir)
misc.copy_file('%s/data/local/dict/optional_silence.txt' % src_model_dir,
'%s/data/local/dict/optional_silence.txt' % work_dir)
misc.copy_file('%s/data/local/dict/extra_questions.txt' % src_model_dir,
'%s/data/local/dict/extra_questions.txt' % work_dir)
#
# language model
#
misc.copy_file(lm_fn, '%s/lm.arpa' % work_dir)
#
# create skeleton dst model
#
logging.info("creating skeleton destination model...")
misc.mkdirs('%s/exp/adapt' % work_dir)
misc.copy_file('%s/model/final.mdl' % src_model_dir,
'%s/exp/adapt/final.mdl' % work_dir)
misc.copy_file('%s/model/cmvn_opts' % src_model_dir,
'%s/exp/adapt/cmvn_opts' % work_dir)
misc.copy_file('%s/model/tree' % src_model_dir,
'%s/exp/adapt/tree' % work_dir)
for optional_file in [
'final.mat', 'splice_opts', 'final.occs', 'full.mat'
]:
if os.path.exists('%s/model/%s' % (src_model_dir, optional_file)):
misc.copy_file('%s/model/%s' % (src_model_dir, optional_file),
'%s/exp/adapt/%s' % (work_dir, optional_file))
if os.path.exists('%s/extractor' % src_model_dir):
misc.mkdirs('%s/exp/extractor' % work_dir)
misc.copy_file('%s/extractor/final.mat' % src_model_dir,
'%s/exp/extractor/final.mat' % work_dir)
misc.copy_file('%s/extractor/global_cmvn.stats' % src_model_dir,
'%s/exp/extractor/global_cmvn.stats' % work_dir)
misc.copy_file('%s/extractor/final.dubm' % src_model_dir,
'%s/exp/extractor/final.dubm' % work_dir)
misc.copy_file('%s/extractor/final.ie' % src_model_dir,
'%s/exp/extractor/final.ie' % work_dir)
misc.copy_file('%s/extractor/splice_opts' % src_model_dir,
'%s/exp/extractor/splice_opts' % work_dir)
misc.mkdirs('%s/exp/ivectors_test_hires/conf' % work_dir)
misc.copy_file(
'%s/ivectors_test_hires/conf/splice.conf' % src_model_dir,
'%s/exp/ivectors_test_hires/conf' % work_dir)
misc.mkdirs('%s/conf' % work_dir)
misc.copy_file('%s/conf/mfcc.conf' % src_model_dir,
'%s/conf/mfcc.conf' % work_dir)
misc.copy_file('%s/conf/mfcc_hires.conf' % src_model_dir,
'%s/conf/mfcc_hires.conf' % work_dir)
misc.copy_file('%s/conf/online_cmvn.conf' % src_model_dir,
'%s/conf/online_cmvn.conf' % work_dir)
#
# copy scripts and config files
#
misc.copy_file('%s/kaldi-run-adaptation.sh' % tmpl_dir,
'%s/run-adaptation.sh' % work_dir)
misc.copy_file('%s/kaldi-cmd.sh' % tmpl_dir, '%s/cmd.sh' % work_dir)
misc.render_template('%s/kaldi-path.sh.template' % tmpl_dir,
'%s/path.sh' % work_dir,
kaldi_root=kaldi_root)
misc.copy_file('%s/kaldi-model-dist.sh' % tmpl_dir,
'%s/model-dist.sh' % work_dir)
misc.symlink('%s/egs/wsj/s5/steps' % kaldi_root, '%s/steps' % work_dir)
misc.symlink('%s/egs/wsj/s5/utils' % kaldi_root, '%s/utils' % work_dir)
cmd = '/bin/bash -c "pushd %s && bash run-adaptation.sh && popd"' % work_dir
logging.info(cmd)
os.system(cmd)
cmd = '/bin/bash -c "pushd %s && bash model-dist.sh "%s" && popd"' % (
work_dir, dst_model_name)
logging.info(cmd)
os.system(cmd)
# vm objs
vm_srcs = ['vm/vm.c']
vm_objs = tools.cons_objs(vm_srcs, vm_hs + clib_hs)
vm_objs = vm_objs + vm_support_objs
# Construct the convenience test binary
test_objs = tools.cons_objs(['vm/test.c'], vm_hs + clib_hs)
test_bin = tools.cons_bin('bin/testvm',
test_objs + vm_support_objs + clib_objs, [])
# Construct VM object
tools.link_objs(vm_objs + clib_objs, "product/vm.o")
include_headers = clib_hs + vm_hs + ['modules/modules.h', 'lang/lang.h']
for header in include_headers:
tools.copy_file('product/include/' + header, header)
##############################################################################
# WRITE META FILE
meta = 'cflags={0}\nlflags={1}\nmodules={2}\n'.format(
" ".join(conf["CFLAGS"]),
" ".join(conf["LFLAGS"]),
" ".join(DEF_MODULES))
rewrite_meta = False
if not os.path.exists('product/ripe.meta'):
rewrite_meta = True
else:
f = open('product/ripe.meta', 'r')
contents = f.read()
# vm objs
vm_srcs = ['vm/vm.c']
vm_objs = tools.cons_objs(vm_srcs, vm_hs + clib_hs)
vm_objs = vm_objs + vm_support_objs
# Construct the convenience test binary
test_objs = tools.cons_objs(['vm/test.c'], vm_hs + clib_hs)
test_bin = tools.cons_bin('bin/testvm',
test_objs + vm_support_objs + clib_objs, [])
# Construct VM object
tools.link_objs(vm_objs + clib_objs, "product/vm.o")
include_headers = clib_hs + vm_hs + ['modules/modules.h', 'lang/lang.h']
for header in include_headers:
tools.copy_file('product/include/' + header, header)
##############################################################################
# WRITE META FILE
meta = 'cflags={0}\nlflags={1}\nmodules={2}\n'.format(" ".join(conf["CFLAGS"]),
" ".join(conf["LFLAGS"]),
" ".join(DEF_MODULES))
rewrite_meta = False
if not os.path.exists('product/ripe.meta'):
rewrite_meta = True
else:
f = open('product/ripe.meta', 'r')
contents = f.read()
f.close()
name = input('Enter file or folder name: ')
if name == '':
print('File or folder name is missing')
save_info('Error delete - File or folder name is missing')
else:
delete_file(name)
save_info(f'Deleted {name}')
elif command == '4':
name = input('Enter file or folder to copy: ')
new_name = input('Enter the new destination: ')
if name == '':
print('File or folder name is missing')
save_info('Error copy - File or folder name is missing')
else:
try:
copy_file(name, new_name)
except FileNotFoundError:
print('New destination is missing')
save_info('Error copy - New destination is missing')
else:
copy_file(name, new_name)
save_info(f'Succes copy. From {name} to {new_name}')
elif command == '5':
game()
elif command == 'help':
print('list - show files and folders')
print('0 - change working directory')
print('1 - create file')
print('2 - create folder')
print('3 - delete file or folder')
print('4 - copy file or folder')
def run_post_steps(component_data_dict, parent_function):
parent_function()
# Get required paths
ssh_filename = os.path.join(
component_data_dict["lfsbuilder_src_directory"], "recipes",
"components", "openssh", "files",
component_data_dict["openssh_public_key_filename"])
# .- get $HOME directory path
if component_data_dict["openssh_username"] == "root":
# It can be dangerous!
printer.warning("WARNING: will configure SSH access for 'root'")
home_directory = os.path.join(config.BASE_DIRECTORY, "root")
elif component_data_dict[
"openssh_username"] == "config.NON_PRIVILEGED_USERNAME":
# Update dictionary value
tools.add_to_dictionary(component_data_dict,
"openssh_username",
config.NON_PRIVILEGED_USERNAME,
concat=False)
# Set home directory
home_directory = os.path.join(config.BASE_DIRECTORY, "home",
config.NON_PRIVILEGED_USERNAME)
else:
home_directory = os.path.join(config.BASE_DIRECTORY, "home",
component_data_dict["openssh_username"])
# .- '$HOME/.ssh' path
ssh_config_path = os.path.join(home_directory, ".ssh")
# .- destination file path
ssh_destination_filename = os.path.join(
ssh_config_path, component_data_dict["openssh_public_key_filename"])
# .- authorized_keys
authorized_keys = os.path.join(ssh_config_path, "authorized_keys")
if os.path.exists(ssh_filename) is False:
# Do not configure. SSH public key do not exists.
msg = """WARNING: SSH access will not be configured because \
the provided public key file '{k}' do not exists."""
msg = msg.format(k=component_data_dict["openssh_public_key_filename"])
printer.warning(msg)
elif tools.check_chroot_user_exists(
component_data_dict["openssh_username"]) is False:
# Do not configure. SSH username do not exists.
msg = """WARNING: SSH access will not be configured because \
the provided username '{u}' do not exists."""
msg = msg.format(u=component_data_dict["openssh_username"])
printer.warning(msg)
elif os.path.exists(home_directory) is False:
# Do not configure. SSH username's home directory do not exists.
msg = """WARNING: SSH access will not be configured because \
the home directory '{h}' do not exists."""
msg = msg.format(h=home_directory)
printer.warning(msg)
else:
msg = "Installing provided SSH public key '{k}' for username '{u}'"
msg = msg.format(k=component_data_dict["openssh_public_key_filename"],
u=component_data_dict["openssh_username"])
printer.substep_info(msg)
# .- create 'ssh_config_path' directory
tools.create_directory(ssh_config_path)
# .- copy public key file
tools.copy_file(ssh_filename, ssh_destination_filename)
# .- add to authorized keys
header = "# --- {f} ---".format(
f=component_data_dict["openssh_public_key_filename"])
tools.add_text_to_file(authorized_keys, header)
tools.add_text_to_file(authorized_keys, tools.read_file(ssh_filename))
# .- get 'UID' and 'GID' values to set permission
etc_passwd_values = tools.get_uid_gid_chroot_username(
component_data_dict["openssh_username"])
# .- set 'ssh_config_path' permission
tools.set_numeric_recursive_owner_and_group(ssh_config_path,
etc_passwd_values["uid"],
etc_passwd_values["gid"])
