def copy_once_file(self, paths): """ Copy one of package files (this method is called from `copy_files` method) """ if os.path.isfile(paths[1]): if paths[0] in self.conffiles: self.copied_files.append('cf:' + paths[0]) old_conffiles = [item[-1] for item in self.old_conffiles] if paths[0] in old_conffiles: f_hash = calc_file_sha256(paths[1]) if [f_hash, paths[0]] in self.old_conffiles: self.uncopied_conffiles[paths[0]] = f_hash return else: if self.keep_conffiles: self.uncopied_conffiles[paths[0]] = f_hash return else: self.copied_files.append('f:' + paths[0]) os.system('cp "' + paths[1] + '" "' + Env.base_path(paths[0]) + '"') else: os.mkdir(Env.base_path(paths[0])) if paths[1] in self.conffiles: self.copied_files.append('cd:' + paths[0]) else: self.copied_files.append('d:' + paths[0])
def run(): """ start running tests """ print('Starting test system...') print('=======================') # load test environment print('Loading test environment...', end=' ') load_test_env() print(ansi.green + 'created in ' + Env.base_path() + ansi.reset) print() # enable testing mode pr.is_testing = True SysArch.is_testing = True # load tests list tests_list = os.listdir('tests/items') # clean up tests list orig_tests = [] for test in tests_list: if test[len(test)-3:] == '.py': exec('from items.' + test[:len(test)-3] + ' import ' + test[:len(test)-3]) exec("orig_tests.append(" + test[:len(test)-3] + "())") # start running tests count = 0 for test in orig_tests: test_name = test.get_name() print('\t[' + str(count + 1) + '/' + str(len(orig_tests)) + ']\t' + test_name.replace('_', ' ') + ': ', end='', flush=True) test.refresh_env() test.run() test.refresh_env() print(ansi.green + 'PASS' + ansi.reset) count += 1 print() print(ansi.green + 'All ' + str(count) + ' tests passed successfuly') print('Cleaning up...' + ansi.reset) if os.path.isfile('testpkgc-1.0.cati'): os.remove('testpkgc-1.0.cati') shutil.rmtree(Env.base_path_dir) Temp.clean()
def run(self): """ Run command """ # require root permission require_root_permission() result_code = 0 packages_to_reinstall = [] if not self.is_quiet(): pr.p('Starting checking system health and security...') pr.p('===============================================') # check state state_cmd = StateCommand() out = state_cmd.handle(ArgParser.parse(['cati', 'state'])) if out > 0: return out # search for conflict and dependency corruptions if not self.is_quiet(): pr.p('Checking dependency and conflict corruptions...') dependency_problems = [] conflict_problems = [] installed_packages = Pkg.installed_list()['list'] for pkg in installed_packages: if self.is_verbose(): pr.p('[info] checking dependencies and conflicts for ' + pkg.data['name'] + '...') for dp in pkg.get_depends(): if not Pkg.check_state(dp): dependency_problems.append([ pkg, dp ]) for conflict in pkg.get_conflicts(): if Pkg.check_state(conflict): conflict_problems.append([ pkg, conflict ]) if dependency_problems or conflict_problems: for depend in dependency_problems: pr.p(ansi.red + 'dependency problem for ' + depend[0].data['name'] + ': ' + depend[1] + ansi.reset) packages_to_reinstall.append(depend[0]) for conflict in conflict_problems: pr.p(ansi.red + 'conflict problem for ' + conflict[0].data['name'] + ': ' + conflict[1] + ansi.reset) packages_to_reinstall.append(conflict[0]) result_code = 1 else: pr.p(ansi.green + 'There is not any conflict or dependnecy problem and everything is ok' + ansi.reset) # check static files if not self.is_quiet(): pr.p('Checking packages static files...') staticfile_problems = [] for pkg in installed_packages: if self.is_verbose(): pr.p('[info] checking static files for ' + pkg.data['name'] + '...') files = pkg.installed_static_files() for f in files: f[1] = Env.base_path(f[1]) if os.path.isfile(f[1]): wanted_hash = f[0] current_hash = calc_file_sha256(f[1]) if wanted_hash != current_hash: staticfile_problems.append([ pkg, f, 'file is changed' ]) else: staticfile_problems.append([ pkg, f, 'file is deleted' ]) if staticfile_problems: for problem in staticfile_problems: pr.p(ansi.red + 'staticfile problem in package ' + problem[0].data['name'] + ': ' + problem[1][1] + ': ' + problem[2] + ansi.reset) packages_to_reinstall.append(problem[0]) result_code = 1 else: pr.p(ansi.green + 'all of static files are ok' + ansi.reset) # check repos config files health if not self.is_quiet(): pr.p('Checking cati configuration files...') if self.is_verbose(): pr.p('[info] checking repositories config...') repos = Repo.get_list() pr.p(ansi.red, end='') ReposListErrorShower.show(repos) pr.p(ansi.reset, end='') is_any_repo_error = False for repo in repos: if repo.syntax_errors: is_any_repo_error = True result_code = 1 if not is_any_repo_error: pr.p(ansi.green + 'all of cati configuration files are ok' + ansi.reset) # check database files if not self.is_quiet(): pr.p('Checking cati database...') database_problems = [] for f in os.listdir(Env.installed_lists()): if self.is_verbose(): pr.p('[info] checking database install dir for ' + f + '...') if not os.path.isfile(Env.installed_lists('/' + f + '/files')) or not os.path.isfile(Env.installed_lists('/' + f + '/ver')): database_problems.append('installed packages database: directory ' + Env.installed_lists('/' + f) + ' is corrupt') for f in os.listdir(Env.security_blacklist()): if self.is_verbose(): pr.p('[info] checking security blacklist part ' + f + '...') if not os.path.isfile(Env.security_blacklist('/' + f)): database_problems.append('security blacklist: an directory detected: ' + Env.security_blacklist('/' + f)) else: tmp = open(Env.security_blacklist('/' + f), 'r') try: json.loads(tmp.read()) except: database_problems.append('security blacklist: invalid json data in ' + Env.security_blacklist('/' + f)) if database_problems: for problem in database_problems: pr.p(ansi.red + 'database: ' + problem + ansi.reset) result_code = 1 else: pr.p(ansi.green + 'all of cati database is ok' + ansi.reset) if not self.is_quiet(): if packages_to_reinstall: pr.p(ansi.blue + 'We suggest re-install this packages:') for pkg in packages_to_reinstall: pr.p('- ' + pkg.data['name']) if not self.has_option('--autofix'): pr.p('use --autofix option to re-install them or do this manually') pr.p(ansi.reset, end='') else: pr.p(ansi.reset, end='') packages_names = [pkg.data['name'] for pkg in packages_to_reinstall] install_cmd = InstallCommand() args = ['cati', 'install', '--reinstall', *packages_names] cmd_str = '' for arg in args: cmd_str += arg + ' ' cmd_str = cmd_str.strip() pr.p(cmd_str) return install_cmd.handle(ArgParser.parse(args)) return result_code
def env(self, path=''): return Env.base_path(path)
def run(pkg: Pkg, events: dict, remove_conffiles=False, run_scripts=True): """ Remove pkg """ events['removing_package'](pkg) # run rm-before script if run_scripts: if os.path.isfile( Env.installed_lists('/' + pkg.data['name'] + '/rm-before')): os.system('chmod +x "' + Env.installed_lists('/' + pkg.data['name'] + '/rm-before') + '"') with_conffiles_arg = 'without-conffiles' if remove_conffiles: with_conffiles_arg = 'with-conffiles' os.system( Env.installed_lists('/' + pkg.data['name'] + '/rm-before') + ' ' + with_conffiles_arg) # remove package installed_files = open( Env.installed_lists('/' + pkg.data['name'] + '/files'), 'r').read() installed_files = installed_files.strip().split('\n') for f in list(reversed(installed_files)): if f != '': f_type = f.strip().split(':', 1)[0] f_path = f.strip().split(':', 1)[1] if f_type == 'f': if os.path.isfile(Env.base_path(f_path)): os.remove(Env.base_path(f_path)) elif f_type == 'd': try: os.rmdir(Env.base_path(f_path)) except: events['dir_is_not_empty'](pkg, f) elif f_type == 'cf': if remove_conffiles: if os.path.isfile(Env.base_path(f_path)): os.remove(Env.base_path(f_path)) else: Remove.add_to_unremoved_conffiles(pkg, f_path) elif f_type == 'cd': if remove_conffiles: try: os.rmdir(Env.base_path(f_path)) except: events['dir_is_not_empty'](pkg, f) else: Remove.add_to_unremoved_conffiles(pkg, f_path) # run rm-after script if run_scripts: if os.path.isfile( Env.installed_lists('/' + pkg.data['name'] + '/rm-after')): with_conffiles_arg = 'without-conffiles' if remove_conffiles: with_conffiles_arg = 'with-conffiles' os.system('chmod +x "' + Env.installed_lists('/' + pkg.data['name'] + '/rm-after') + '"') os.system( Env.installed_lists('/' + pkg.data['name'] + '/rm-after') + ' ' + with_conffiles_arg) # remove installation config shutil.rmtree(Env.installed_lists('/' + pkg.data['name'])) # remove any script if os.path.isfile(Env.any_scripts('/' + pkg.data['name'])): os.remove(Env.any_scripts('/' + pkg.data['name'])) events['package_remove_finished'](pkg)
def copy_files(self, pkg: BaseArchive, directory_not_empty_event, target_path='') -> list: """ Copy package files on system Args: pkg (BaseArchive): the package archive object directory_not_empty_event (callable): the function will be run when we want to delete old direcotry of package and that is not empty. target_path (str): target path prefix of copied files location. default is `/` means copies files on the root directory. you can change it. Returns: list[str]: list of copied files """ # load package old files list old_files = [] if os.path.isfile( Env.installed_lists('/' + pkg.data['name'] + '/files')): try: f = open( Env.installed_lists('/' + pkg.data['name'] + '/files'), 'r') for line in f.read().strip().split('\n'): if line != '': old_files.append(line.strip()) except: pass old_files = list(reversed(old_files)) # load unremoved conffiles list unremoved_conffiles_f = open(Env.unremoved_conffiles(), 'r') unremoved_conffiles = unremoved_conffiles_f.read().strip().split('\n') unremoved_conffiles_f.close() temp_dir = self.extracted_package_dir # load files list from `files` directory of package self.loaded_files = [] self.load_files(temp_dir + '/files', temp_dir + '/files') self.loaded_files = [[target_path + f[0], f[1]] for f in self.loaded_files] # check file conflicts all_installed_files = Pkg.get_all_installed_files_list() for lf in self.loaded_files: if not os.path.isdir(lf[1]): for insf in all_installed_files: if insf[0].split(':', 1)[0] != pkg.data['name']: if lf[0] == insf[2]: insf_pkg = Pkg.load_last(insf[0]) if insf_pkg: insf[0] = insf[0] + ':' + insf_pkg.installed() # check package is in `replaces` list do_raise_error = True replaces_list = pkg.get_replaces() for rep in replaces_list: if Pkg.check_state(rep): tmp_parts = rep.split(' ') tmp_parts = tmp_parts[0].split('>') tmp_parts = tmp_parts[0].split('<') tmp_parts = tmp_parts[0].split('=') tmp_parts = tmp_parts[0] if tmp_parts == insf[0].split(':', 1)[0]: do_raise_error = False if do_raise_error: raise FileConflictError('package ' + pkg.data['name'] + ':' + pkg.data['version'] + ' and ' + insf[0] + ' both has file "' + lf[0] + '"') # copy loaded files self.copied_files = [] for f in self.loaded_files: if os.path.exists(Env.base_path(f[0])): if os.path.isfile(Env.base_path(f[0])): if ('f:' + f[0]) in old_files or ('cf:' + f[0]) in old_files: self.copy_once_file(f) try: old_files.pop(old_files.index(('f:' + f[0]))) except: pass else: if f[0] in unremoved_conffiles: self.copy_once_file(f) unremoved_conffiles.pop( unremoved_conffiles.index(f[0])) else: self.copy_once_file(f) else: if ('d:' + f[0]) in old_files or ('cd:' + f[0]) in old_files: if ('cd:' + f[0]) in old_files: self.copied_files.append('cd:' + f[0]) old_files.pop(old_files.index(('cd:' + f[0]))) else: self.copied_files.append('d:' + f[0]) old_files.pop(old_files.index(('d:' + f[0]))) else: if f[0] in unremoved_conffiles: self.copied_files.append('d:' + f[0]) unremoved_conffiles.pop( unremoved_conffiles.index(f[0])) else: self.copy_once_file(f) # delete not wanted old files for item in old_files: parts = item.strip().split(':', 1) if parts[0] == 'cf' or parts[0] == 'cd': pass else: if os.path.isfile(parts[1]): os.remove(parts[1]) else: try: os.rmdir(parts[1]) except: # directory is not emptyr directory_not_empty_event(pkg, parts[1]) # write new unremoved conffiles list unremoved_conffiles_f = open(Env.unremoved_conffiles(), 'w') new_content = '' for item in unremoved_conffiles: new_content += item + '\n' unremoved_conffiles_f.write(new_content) unremoved_conffiles_f.close() return self.copied_files
def install(self, pkg: BaseArchive, index_updater_events: dict, installer_events: dict, is_manual=True, run_scripts=True, target_path='', keep_conffiles=False, check_security_blacklist=True): """ Install .cati package Args: pkg (BaseArchive): the package archive object index_updater_events (dict): events will be passed to `dotcati.ListUpdater.update_indexes()` installer_events (dict): The events - package_currently_install: gets the current installed version - package_new_installs: gets package archive - package_installed: will call after package installation - dep_and_conflict_error: will run when there is depends or conflict error - arch_error: will run when package arch is not sync with sys arch is_manual (bool): installed package as manual or not (default is True means manual) run_scripts (bool): run package install scripts or not (default is True) target_path (str): where is the target path for installed files (will pass to `self.copy_files()`) keep_conffiles (bool): stil keep config files if changed (default is True) check_security_blacklist (bool): check package is in security blacklist or not """ self.conffiles = pkg.get_conffiles() self.pkg = pkg self.keep_conffiles = keep_conffiles self.uncopied_conffiles = {} # check package is in security blacklist if check_security_blacklist: self.check_security_blacklist(pkg) # check package architecture if not pkg.data['arch'] in SysArch.allowed_archs(): return installer_events['arch_error'](pkg) # add package data to lists self.add_package_to_lists(pkg, index_updater_events) # check package dependencies and conflicts try: self.check_dep_and_conf(pkg) except DependencyError as ex: return installer_events['dep_and_conflict_error'](pkg, ex) except ConflictError as ex: return installer_events['dep_and_conflict_error'](pkg, ex) # load old conffiles self.old_conffiles = [] try: f = open( Env.installed_lists('/' + pkg.data['name'] + '/conffiles'), 'r') content = f.read() f.close() tmp = content.strip().split('\n') self.old_conffiles = [item.strip().split('@') for item in tmp] except: pass # extract package in a temp place temp_dir = Temp.make_dir() os.rmdir(temp_dir) try: pkg.extractall(temp_dir) except IsADirectoryError: pass self.extracted_package_dir = temp_dir # install package if Pkg.is_installed(pkg.data['name']): installer_events['package_currently_installed']( pkg, Pkg.installed_version(pkg.data['name'])) else: installer_events['package_new_installs'](pkg) if run_scripts: self.run_script('ins-before') copied_files = self.copy_files(pkg, installer_events['directory_not_empty'], target_path) # set install configuration if not os.path.isdir(Env.installed_lists('/' + pkg.data['name'])): os.mkdir(Env.installed_lists('/' + pkg.data['name'])) f_ver = open(Env.installed_lists('/' + pkg.data['name'] + '/ver'), 'w') f_ver.write(pkg.data['version']) # write installed version f_ver.close() # write copied files list f_files = open(Env.installed_lists('/' + pkg.data['name'] + '/files'), 'w') copied_files_str = '' for copied_file in copied_files: copied_files_str += copied_file + '\n' f_files.write(copied_files_str.strip()) # write copied files f_files.close() # write conffiles list f_conffiles = open( Env.installed_lists('/' + pkg.data['name'] + '/conffiles'), 'w') copied_conffiles_str = '' for copied_conffile in copied_files: if copied_conffile.split(':')[0] == 'cf': try: conffile_hash = self.uncopied_conffiles[ copied_conffile.split(':', 1)[-1]] except: conffile_hash = calc_file_sha256( Env.base_path(copied_conffile.split(':', 1)[-1])) copied_conffiles_str += conffile_hash + '@' + copied_conffile.split( ':', 1)[-1] + '\n' f_conffiles.write( copied_conffiles_str.strip()) # write copied conffiles f_conffiles.close() # copy `any` script if os.path.isfile(self.extracted_package_dir + '/scripts/any'): os.system('cp "' + self.extracted_package_dir + '/scripts/any' + '" "' + Env.any_scripts('/' + pkg.data['name']) + '"') # save static files list static_files_list = pkg.get_static_files() f_static_files = open( Env.installed_lists('/' + pkg.data['name'] + '/staticfiles'), 'w') static_files_str = '' for copied_file in copied_files: copied_file_path = copied_file.split(':', 1)[1] if copied_file_path in static_files_list: if os.path.isfile(Env.base_path('/' + copied_file_path)): # calculate file sha256 sum copied_file_sha256 = calc_file_sha256( Env.base_path('/' + copied_file_path)) # add file to list static_files_str += copied_file_sha256 + '@' + copied_file_path + '\n' f_static_files.write(static_files_str.strip()) # write copied files f_static_files.close() f_installed_at = open( Env.installed_lists('/' + pkg.data['name'] + '/installed_at'), 'w') f_installed_at.write(str(time.time())) # write time (installed at) f_installed_at.close() if is_manual: f_manual = open( Env.installed_lists('/' + pkg.data['name'] + '/manual'), 'w') f_manual.write('') f_manual.close() if run_scripts: self.run_script('ins-after') # copy remove scripts if os.path.isfile(self.extracted_package_dir + '/scripts/rm-before'): os.system('cp "' + self.extracted_package_dir + '/scripts/rm-before' + '" "' + Env.installed_lists('/' + pkg.data['name'] + '/rm-before') + '"') if os.path.isfile(self.extracted_package_dir + '/scripts/rm-after'): os.system('cp "' + self.extracted_package_dir + '/scripts/rm-after' + '" "' + Env.installed_lists('/' + pkg.data['name'] + '/rm-after') + '"') # pop package from state BaseTransaction.pop_state() # call package installed event installer_events['package_installed'](pkg)