def build_module(module, required): import os.path tools.mkdir_safe('product/modules/%s' % module) out = 'product/modules/%s/%s.o' % (module, module) srcs = [] metafile = 'modules/%s/%s.meta' % (module, module) if os.path.exists(metafile): tools.copy_file('product/modules/%s/%s.meta' % (module, module), metafile) srcs.append(metafile) meta = tools.load_meta(metafile) extra_objs = '' if 'objs' in meta: extra_objs = meta['objs'] src = 'modules/%s/%s.rip' % (module, module) if tools.depends(out, module_deps + [src]): tools.pprint('MOD', src, out) args = ['product/ripe', conf["RFLAGS"], '-n', module, '-c', srcs, src, extra_objs, '-o', out] # Required (default) packages have already been typed, and are # loaded by default. Hence, they do not need to be typed. if required: args.append('--omit-typing') if conf["VERBOSITY"] > 1: args.append('-v') if required: tools.call(args) else: if not tools.try_call(args): failed_modules.append(module)
def create_setenv_script(self): """ Create the 'setenv.sh' file from the 'templates/setenv.tpl' template under the lfsbuilder source directory. """ setenv_filename = os.path.join( self.builder_data_dict["setenv_directory"], self.builder_data_dict["setenv_filename"]) template = os.path.join( self.builder_data_dict["lfsbuilder_templates_directory"], self.builder_data_dict["setenv_template"]) substitution_list = [ "@@LFS_BUILDER_NAME@@", self.builder_data_dict["name"], "@@LFS_ENV_PATH_VALUE@@", self.builder_data_dict["env_PATH_value"], "@@LFS_SOURCES_DIRECTORY@@", self.builder_data_dict["sources_directory"], "@@LFS_TOOLS_DIRECTROY@@", self.builder_data_dict["tools_directory"], "@@LFSBUILDER_SRC_DIRECTORY@@", self.builder_data_dict["lfsbuilder_src_directory"], "@@LFS_BASE_DIRECTORY@@", config.BASE_DIRECTORY, "@@LFS_MAKEFLAGS@@", config.MAKEFLAGS ] # Copy template tools.copy_file(template, setenv_filename) # Substitute parameters tools.substitute_multiple_in_file(setenv_filename, substitution_list)
def build_module(module, required): import os.path tools.mkdir_safe('product/modules/%s' % module) out = 'product/modules/%s/%s.o' % (module, module) srcs = [] metafile = 'modules/%s/%s.meta' % (module, module) if os.path.exists(metafile): tools.copy_file('product/modules/%s/%s.meta' % (module, module), metafile) srcs.append(metafile) meta = tools.load_meta(metafile) extra_objs = '' if 'objs' in meta: extra_objs = meta['objs'] src = 'modules/%s/%s.rip' % (module, module) if tools.depends(out, module_deps + [src]): tools.pprint('MOD', src, out) args = [ 'product/ripe', conf["RFLAGS"], '-n', module, '-c', srcs, src, extra_objs, '-o', out ] # Required (default) packages have already been typed, and are # loaded by default. Hence, they do not need to be typed. if required: args.append('--omit-typing') if conf["VERBOSITY"] > 1: args.append('-v') if required: tools.call(args) else: if not tools.try_call(args): failed_modules.append(module)
def copy_script_header(self, script_filename): """ Copy script header from the 'script.tpl' template under the 'templates' directory. """ template = os.path.join(self.component_data_dict["lfsbuilder_templates_directory"], self.component_data_dict["script_template"]) printer.substep_info("Generating script '{f}'".format(f=script_filename)) tools.copy_file(template, script_filename)
def clean_up(self): '''Move exception files into the exception folder.''' dont_rename = self.parse_failed + self.non_abi tools.move_file(dont_rename, self.fail_folder) for entry in self.exceptions: src = entry[1] dst = os.path.join(self.fail_folder, entry[2]) tools.copy_file(src, dst) os.remove(entry[1]) assert (os.listdir(self.temp_dir), []) shutil.rmtree(self.temp_dir)
def clean_up(self): '''Move exception files into the exception folder.''' dont_rename = self.parse_failed + self.non_abi tools.move_file(dont_rename, self.fail_folder) for entry in self.exceptions: src = entry[1] dst = os.path.join(self.fail_folder, entry[2]) tools.copy_file(src, dst) os.remove(entry[1]) assert(os.listdir(self.temp_dir), []) shutil.rmtree(self.temp_dir)
def run_previous_steps(component_data_dict, parent_function): # Call parent function parent_function() print("Copying custom \'.config\' file") filename = os.path.join(component_data_dict["lfsbuilder_src_directory"], "recipes", "components", "kernel", "files", component_data_dict["kernel_config_filename"]) tools.copy_file( filename, os.path.join(component_data_dict["extracted_directory"], ".config"))
def copy_to_organized_directory(dir_src: str, dir_dst: str) -> bool: """copy source directory to organized directory Get content from source directory and copy to a new directory into new folders with the extentions name Arguments: dir_src {str} -- source directory dir_dst {str} -- destination directory Raises: IOError: path dir_src not found IOError: path dir_dst not found Returns: bool -- if successful return True """ if not checks.check_exists(dir_src): raise IOError("Directory '{0}' not found.".format(dir_src)) if not checks.check_exists(dir_dst): raise IOError("Directory '{0}' not found.".format(dir_dst)) # copy files temp to new directory list_files = get_files(dir_src) for file in list_files: basename = os.path.basename(file) search = re.search(r"\.([A-Za-z0-9]+)$", basename) if search: extension = search.group(1) # check = [ext for ext in extensions if search.group(1) == ext] directory_extension = os.path.join(dir_dst, extension) if not checks.check_exists(directory_extension): tools.create_directory(directory_extension) # build target directory target_directory = os.path.join(directory_extension, basename) # copy file tools.copy_file(file, target_directory) return True
def split_dataset(rate): if not os.path.exists(train_dir): os.makedirs(train_dir) if not os.path.exists(val_dir): os.makedirs(val_dir) files = os.listdir(org_dir) for name in files: tools.copy_file(org_dir + name, train_dir + name) file_num = len(files) pick_num = int(file_num * rate) print(file_num) print(pick_num) sample = random.sample(files, pick_num) for name in sample: tools.move_file(train_dir + name, val_dir + name) return
def kaldi_adapt_lm(kaldi_root, src_model_dir, lm_fn, work_dir, dst_model_name): steps_path = '%s/egs/wsj/s5/steps' % kaldi_root if not os.path.exists(steps_path): raise Exception( '%s does not exist - is kaldi really installed in %s ?' % (steps_path, kaldi_root)) tmpl_dir = os.path.dirname(os.path.abspath(__file__)) + '/templates' # # copy dictionary and phoneme sets from original model # logging.info("copying dictionary and phoneme sets from original model...") misc.mkdirs('%s/data/local/dict' % work_dir) misc.copy_file('%s/data/local/dict/lexicon.txt' % src_model_dir, '%s/data/local/dict/lexicon.txt' % work_dir) misc.copy_file('%s/data/local/dict/nonsilence_phones.txt' % src_model_dir, '%s/data/local/dict/nonsilence_phones.txt' % work_dir) misc.copy_file('%s/data/local/dict/silence_phones.txt' % src_model_dir, '%s/data/local/dict/silence_phones.txt' % work_dir) misc.copy_file('%s/data/local/dict/optional_silence.txt' % src_model_dir, '%s/data/local/dict/optional_silence.txt' % work_dir) misc.copy_file('%s/data/local/dict/extra_questions.txt' % src_model_dir, '%s/data/local/dict/extra_questions.txt' % work_dir) # # language model # misc.copy_file(lm_fn, '%s/lm.arpa' % work_dir) # # create skeleton dst model # logging.info("creating skeleton destination model...") misc.mkdirs('%s/exp/adapt' % work_dir) misc.copy_file('%s/model/final.mdl' % src_model_dir, '%s/exp/adapt/final.mdl' % work_dir) misc.copy_file('%s/model/cmvn_opts' % src_model_dir, '%s/exp/adapt/cmvn_opts' % work_dir) misc.copy_file('%s/model/tree' % src_model_dir, '%s/exp/adapt/tree' % work_dir) for optional_file in [ 'final.mat', 'splice_opts', 'final.occs', 'full.mat' ]: if os.path.exists('%s/model/%s' % (src_model_dir, optional_file)): misc.copy_file('%s/model/%s' % (src_model_dir, optional_file), '%s/exp/adapt/%s' % (work_dir, optional_file)) if os.path.exists('%s/extractor' % src_model_dir): misc.mkdirs('%s/exp/extractor' % work_dir) misc.copy_file('%s/extractor/final.mat' % src_model_dir, '%s/exp/extractor/final.mat' % work_dir) misc.copy_file('%s/extractor/global_cmvn.stats' % src_model_dir, '%s/exp/extractor/global_cmvn.stats' % work_dir) misc.copy_file('%s/extractor/final.dubm' % src_model_dir, '%s/exp/extractor/final.dubm' % work_dir) misc.copy_file('%s/extractor/final.ie' % src_model_dir, '%s/exp/extractor/final.ie' % work_dir) misc.copy_file('%s/extractor/splice_opts' % src_model_dir, '%s/exp/extractor/splice_opts' % work_dir) misc.mkdirs('%s/exp/ivectors_test_hires/conf' % work_dir) misc.copy_file( '%s/ivectors_test_hires/conf/splice.conf' % src_model_dir, '%s/exp/ivectors_test_hires/conf' % work_dir) misc.mkdirs('%s/conf' % work_dir) misc.copy_file('%s/conf/mfcc.conf' % src_model_dir, '%s/conf/mfcc.conf' % work_dir) misc.copy_file('%s/conf/mfcc_hires.conf' % src_model_dir, '%s/conf/mfcc_hires.conf' % work_dir) misc.copy_file('%s/conf/online_cmvn.conf' % src_model_dir, '%s/conf/online_cmvn.conf' % work_dir) # # copy scripts and config files # misc.copy_file('%s/kaldi-run-adaptation.sh' % tmpl_dir, '%s/run-adaptation.sh' % work_dir) misc.copy_file('%s/kaldi-cmd.sh' % tmpl_dir, '%s/cmd.sh' % work_dir) misc.render_template('%s/kaldi-path.sh.template' % tmpl_dir, '%s/path.sh' % work_dir, kaldi_root=kaldi_root) misc.copy_file('%s/kaldi-model-dist.sh' % tmpl_dir, '%s/model-dist.sh' % work_dir) misc.symlink('%s/egs/wsj/s5/steps' % kaldi_root, '%s/steps' % work_dir) misc.symlink('%s/egs/wsj/s5/utils' % kaldi_root, '%s/utils' % work_dir) cmd = '/bin/bash -c "pushd %s && bash run-adaptation.sh && popd"' % work_dir logging.info(cmd) os.system(cmd) cmd = '/bin/bash -c "pushd %s && bash model-dist.sh "%s" && popd"' % ( work_dir, dst_model_name) logging.info(cmd) os.system(cmd)
# vm objs vm_srcs = ['vm/vm.c'] vm_objs = tools.cons_objs(vm_srcs, vm_hs + clib_hs) vm_objs = vm_objs + vm_support_objs # Construct the convenience test binary test_objs = tools.cons_objs(['vm/test.c'], vm_hs + clib_hs) test_bin = tools.cons_bin('bin/testvm', test_objs + vm_support_objs + clib_objs, []) # Construct VM object tools.link_objs(vm_objs + clib_objs, "product/vm.o") include_headers = clib_hs + vm_hs + ['modules/modules.h', 'lang/lang.h'] for header in include_headers: tools.copy_file('product/include/' + header, header) ############################################################################## # WRITE META FILE meta = 'cflags={0}\nlflags={1}\nmodules={2}\n'.format( " ".join(conf["CFLAGS"]), " ".join(conf["LFLAGS"]), " ".join(DEF_MODULES)) rewrite_meta = False if not os.path.exists('product/ripe.meta'): rewrite_meta = True else: f = open('product/ripe.meta', 'r') contents = f.read()
# vm objs vm_srcs = ['vm/vm.c'] vm_objs = tools.cons_objs(vm_srcs, vm_hs + clib_hs) vm_objs = vm_objs + vm_support_objs # Construct the convenience test binary test_objs = tools.cons_objs(['vm/test.c'], vm_hs + clib_hs) test_bin = tools.cons_bin('bin/testvm', test_objs + vm_support_objs + clib_objs, []) # Construct VM object tools.link_objs(vm_objs + clib_objs, "product/vm.o") include_headers = clib_hs + vm_hs + ['modules/modules.h', 'lang/lang.h'] for header in include_headers: tools.copy_file('product/include/' + header, header) ############################################################################## # WRITE META FILE meta = 'cflags={0}\nlflags={1}\nmodules={2}\n'.format(" ".join(conf["CFLAGS"]), " ".join(conf["LFLAGS"]), " ".join(DEF_MODULES)) rewrite_meta = False if not os.path.exists('product/ripe.meta'): rewrite_meta = True else: f = open('product/ripe.meta', 'r') contents = f.read() f.close()
name = input('Enter file or folder name: ') if name == '': print('File or folder name is missing') save_info('Error delete - File or folder name is missing') else: delete_file(name) save_info(f'Deleted {name}') elif command == '4': name = input('Enter file or folder to copy: ') new_name = input('Enter the new destination: ') if name == '': print('File or folder name is missing') save_info('Error copy - File or folder name is missing') else: try: copy_file(name, new_name) except FileNotFoundError: print('New destination is missing') save_info('Error copy - New destination is missing') else: copy_file(name, new_name) save_info(f'Succes copy. From {name} to {new_name}') elif command == '5': game() elif command == 'help': print('list - show files and folders') print('0 - change working directory') print('1 - create file') print('2 - create folder') print('3 - delete file or folder') print('4 - copy file or folder')
def run_post_steps(component_data_dict, parent_function): parent_function() # Get required paths ssh_filename = os.path.join( component_data_dict["lfsbuilder_src_directory"], "recipes", "components", "openssh", "files", component_data_dict["openssh_public_key_filename"]) # .- get $HOME directory path if component_data_dict["openssh_username"] == "root": # It can be dangerous! printer.warning("WARNING: will configure SSH access for 'root'") home_directory = os.path.join(config.BASE_DIRECTORY, "root") elif component_data_dict[ "openssh_username"] == "config.NON_PRIVILEGED_USERNAME": # Update dictionary value tools.add_to_dictionary(component_data_dict, "openssh_username", config.NON_PRIVILEGED_USERNAME, concat=False) # Set home directory home_directory = os.path.join(config.BASE_DIRECTORY, "home", config.NON_PRIVILEGED_USERNAME) else: home_directory = os.path.join(config.BASE_DIRECTORY, "home", component_data_dict["openssh_username"]) # .- '$HOME/.ssh' path ssh_config_path = os.path.join(home_directory, ".ssh") # .- destination file path ssh_destination_filename = os.path.join( ssh_config_path, component_data_dict["openssh_public_key_filename"]) # .- authorized_keys authorized_keys = os.path.join(ssh_config_path, "authorized_keys") if os.path.exists(ssh_filename) is False: # Do not configure. SSH public key do not exists. msg = """WARNING: SSH access will not be configured because \ the provided public key file '{k}' do not exists.""" msg = msg.format(k=component_data_dict["openssh_public_key_filename"]) printer.warning(msg) elif tools.check_chroot_user_exists( component_data_dict["openssh_username"]) is False: # Do not configure. SSH username do not exists. msg = """WARNING: SSH access will not be configured because \ the provided username '{u}' do not exists.""" msg = msg.format(u=component_data_dict["openssh_username"]) printer.warning(msg) elif os.path.exists(home_directory) is False: # Do not configure. SSH username's home directory do not exists. msg = """WARNING: SSH access will not be configured because \ the home directory '{h}' do not exists.""" msg = msg.format(h=home_directory) printer.warning(msg) else: msg = "Installing provided SSH public key '{k}' for username '{u}'" msg = msg.format(k=component_data_dict["openssh_public_key_filename"], u=component_data_dict["openssh_username"]) printer.substep_info(msg) # .- create 'ssh_config_path' directory tools.create_directory(ssh_config_path) # .- copy public key file tools.copy_file(ssh_filename, ssh_destination_filename) # .- add to authorized keys header = "# --- {f} ---".format( f=component_data_dict["openssh_public_key_filename"]) tools.add_text_to_file(authorized_keys, header) tools.add_text_to_file(authorized_keys, tools.read_file(ssh_filename)) # .- get 'UID' and 'GID' values to set permission etc_passwd_values = tools.get_uid_gid_chroot_username( component_data_dict["openssh_username"]) # .- set 'ssh_config_path' permission tools.set_numeric_recursive_owner_and_group(ssh_config_path, etc_passwd_values["uid"], etc_passwd_values["gid"])