def copy_once_file(self, paths):
""" Copy one of package files (this method is called from `copy_files` method) """
if os.path.isfile(paths[1]):
if paths[0] in self.conffiles:
self.copied_files.append('cf:' + paths[0])
old_conffiles = [item[-1] for item in self.old_conffiles]
if paths[0] in old_conffiles:
f_hash = calc_file_sha256(paths[1])
if [f_hash, paths[0]] in self.old_conffiles:
self.uncopied_conffiles[paths[0]] = f_hash
return
else:
if self.keep_conffiles:
self.uncopied_conffiles[paths[0]] = f_hash
return
else:
self.copied_files.append('f:' + paths[0])
os.system('cp "' + paths[1] + '" "' + Env.base_path(paths[0]) +
'"')
else:
os.mkdir(Env.base_path(paths[0]))
if paths[1] in self.conffiles:
self.copied_files.append('cd:' + paths[0])
else:
self.copied_files.append('d:' + paths[0])
def run():
""" start running tests """
print('Starting test system...')
print('=======================')
# load test environment
print('Loading test environment...', end=' ')
load_test_env()
print(ansi.green + 'created in ' + Env.base_path() + ansi.reset)
print()
# enable testing mode
pr.is_testing = True
SysArch.is_testing = True
# load tests list
tests_list = os.listdir('tests/items')
# clean up tests list
orig_tests = []
for test in tests_list:
if test[len(test)-3:] == '.py':
exec('from items.' + test[:len(test)-3] + ' import ' + test[:len(test)-3])
exec("orig_tests.append(" + test[:len(test)-3] + "())")
# start running tests
count = 0
for test in orig_tests:
test_name = test.get_name()
print('\t[' + str(count + 1) + '/' + str(len(orig_tests)) + ']\t' + test_name.replace('_', ' ') + ': ', end='', flush=True)
test.refresh_env()
test.run()
test.refresh_env()
print(ansi.green + 'PASS' + ansi.reset)
count += 1
print()
print(ansi.green + 'All ' + str(count) + ' tests passed successfuly')
print('Cleaning up...' + ansi.reset)
if os.path.isfile('testpkgc-1.0.cati'):
os.remove('testpkgc-1.0.cati')
shutil.rmtree(Env.base_path_dir)
Temp.clean()
def run(self):
""" Run command """
# require root permission
require_root_permission()
result_code = 0
packages_to_reinstall = []
if not self.is_quiet():
pr.p('Starting checking system health and security...')
pr.p('===============================================')
# check state
state_cmd = StateCommand()
out = state_cmd.handle(ArgParser.parse(['cati', 'state']))
if out > 0:
return out
# search for conflict and dependency corruptions
if not self.is_quiet():
pr.p('Checking dependency and conflict corruptions...')
dependency_problems = []
conflict_problems = []
installed_packages = Pkg.installed_list()['list']
for pkg in installed_packages:
if self.is_verbose():
pr.p('[info] checking dependencies and conflicts for ' + pkg.data['name'] + '...')
for dp in pkg.get_depends():
if not Pkg.check_state(dp):
dependency_problems.append([
pkg, dp
])
for conflict in pkg.get_conflicts():
if Pkg.check_state(conflict):
conflict_problems.append([
pkg, conflict
])
if dependency_problems or conflict_problems:
for depend in dependency_problems:
pr.p(ansi.red + 'dependency problem for ' + depend[0].data['name'] + ': ' + depend[1] + ansi.reset)
packages_to_reinstall.append(depend[0])
for conflict in conflict_problems:
pr.p(ansi.red + 'conflict problem for ' + conflict[0].data['name'] + ': ' + conflict[1] + ansi.reset)
packages_to_reinstall.append(conflict[0])
result_code = 1
else:
pr.p(ansi.green + 'There is not any conflict or dependnecy problem and everything is ok' + ansi.reset)
# check static files
if not self.is_quiet():
pr.p('Checking packages static files...')
staticfile_problems = []
for pkg in installed_packages:
if self.is_verbose():
pr.p('[info] checking static files for ' + pkg.data['name'] + '...')
files = pkg.installed_static_files()
for f in files:
f[1] = Env.base_path(f[1])
if os.path.isfile(f[1]):
wanted_hash = f[0]
current_hash = calc_file_sha256(f[1])
if wanted_hash != current_hash:
staticfile_problems.append([
pkg,
f,
'file is changed'
])
else:
staticfile_problems.append([
pkg,
f,
'file is deleted'
])
if staticfile_problems:
for problem in staticfile_problems:
pr.p(ansi.red + 'staticfile problem in package ' + problem[0].data['name'] + ': ' + problem[1][1] + ': ' + problem[2] + ansi.reset)
packages_to_reinstall.append(problem[0])
result_code = 1
else:
pr.p(ansi.green + 'all of static files are ok' + ansi.reset)
# check repos config files health
if not self.is_quiet():
pr.p('Checking cati configuration files...')
if self.is_verbose():
pr.p('[info] checking repositories config...')
repos = Repo.get_list()
pr.p(ansi.red, end='')
ReposListErrorShower.show(repos)
pr.p(ansi.reset, end='')
is_any_repo_error = False
for repo in repos:
if repo.syntax_errors:
is_any_repo_error = True
result_code = 1
if not is_any_repo_error:
pr.p(ansi.green + 'all of cati configuration files are ok' + ansi.reset)
# check database files
if not self.is_quiet():
pr.p('Checking cati database...')
database_problems = []
for f in os.listdir(Env.installed_lists()):
if self.is_verbose():
pr.p('[info] checking database install dir for ' + f + '...')
if not os.path.isfile(Env.installed_lists('/' + f + '/files')) or not os.path.isfile(Env.installed_lists('/' + f + '/ver')):
database_problems.append('installed packages database: directory ' + Env.installed_lists('/' + f) + ' is corrupt')
for f in os.listdir(Env.security_blacklist()):
if self.is_verbose():
pr.p('[info] checking security blacklist part ' + f + '...')
if not os.path.isfile(Env.security_blacklist('/' + f)):
database_problems.append('security blacklist: an directory detected: ' + Env.security_blacklist('/' + f))
else:
tmp = open(Env.security_blacklist('/' + f), 'r')
try:
json.loads(tmp.read())
except:
database_problems.append('security blacklist: invalid json data in ' + Env.security_blacklist('/' + f))
if database_problems:
for problem in database_problems:
pr.p(ansi.red + 'database: ' + problem + ansi.reset)
result_code = 1
else:
pr.p(ansi.green + 'all of cati database is ok' + ansi.reset)
if not self.is_quiet():
if packages_to_reinstall:
pr.p(ansi.blue + 'We suggest re-install this packages:')
for pkg in packages_to_reinstall:
pr.p('- ' + pkg.data['name'])
if not self.has_option('--autofix'):
pr.p('use --autofix option to re-install them or do this manually')
pr.p(ansi.reset, end='')
else:
pr.p(ansi.reset, end='')
packages_names = [pkg.data['name'] for pkg in packages_to_reinstall]
install_cmd = InstallCommand()
args = ['cati', 'install', '--reinstall', *packages_names]
cmd_str = ''
for arg in args:
cmd_str += arg + ' '
cmd_str = cmd_str.strip()
pr.p(cmd_str)
return install_cmd.handle(ArgParser.parse(args))
return result_code
def env(self, path=''):
return Env.base_path(path)
def run(pkg: Pkg, events: dict, remove_conffiles=False, run_scripts=True):
""" Remove pkg """
events['removing_package'](pkg)
# run rm-before script
if run_scripts:
if os.path.isfile(
Env.installed_lists('/' + pkg.data['name'] +
'/rm-before')):
os.system('chmod +x "' +
Env.installed_lists('/' + pkg.data['name'] +
'/rm-before') + '"')
with_conffiles_arg = 'without-conffiles'
if remove_conffiles:
with_conffiles_arg = 'with-conffiles'
os.system(
Env.installed_lists('/' + pkg.data['name'] +
'/rm-before') + ' ' +
with_conffiles_arg)
# remove package
installed_files = open(
Env.installed_lists('/' + pkg.data['name'] + '/files'),
'r').read()
installed_files = installed_files.strip().split('\n')
for f in list(reversed(installed_files)):
if f != '':
f_type = f.strip().split(':', 1)[0]
f_path = f.strip().split(':', 1)[1]
if f_type == 'f':
if os.path.isfile(Env.base_path(f_path)):
os.remove(Env.base_path(f_path))
elif f_type == 'd':
try:
os.rmdir(Env.base_path(f_path))
except:
events['dir_is_not_empty'](pkg, f)
elif f_type == 'cf':
if remove_conffiles:
if os.path.isfile(Env.base_path(f_path)):
os.remove(Env.base_path(f_path))
else:
Remove.add_to_unremoved_conffiles(pkg, f_path)
elif f_type == 'cd':
if remove_conffiles:
try:
os.rmdir(Env.base_path(f_path))
except:
events['dir_is_not_empty'](pkg, f)
else:
Remove.add_to_unremoved_conffiles(pkg, f_path)
# run rm-after script
if run_scripts:
if os.path.isfile(
Env.installed_lists('/' + pkg.data['name'] + '/rm-after')):
with_conffiles_arg = 'without-conffiles'
if remove_conffiles:
with_conffiles_arg = 'with-conffiles'
os.system('chmod +x "' +
Env.installed_lists('/' + pkg.data['name'] +
'/rm-after') + '"')
os.system(
Env.installed_lists('/' + pkg.data['name'] + '/rm-after') +
' ' + with_conffiles_arg)
# remove installation config
shutil.rmtree(Env.installed_lists('/' + pkg.data['name']))
# remove any script
if os.path.isfile(Env.any_scripts('/' + pkg.data['name'])):
os.remove(Env.any_scripts('/' + pkg.data['name']))
events['package_remove_finished'](pkg)
def copy_files(self,
pkg: BaseArchive,
directory_not_empty_event,
target_path='') -> list:
"""
Copy package files on system
Args:
pkg (BaseArchive): the package archive object
directory_not_empty_event (callable): the function will be run when we want to delete old direcotry
of package and that is not empty.
target_path (str): target path prefix of copied files location. default is `/` means
copies files on the root directory. you can change it.
Returns:
list[str]: list of copied files
"""
# load package old files list
old_files = []
if os.path.isfile(
Env.installed_lists('/' + pkg.data['name'] + '/files')):
try:
f = open(
Env.installed_lists('/' + pkg.data['name'] + '/files'),
'r')
for line in f.read().strip().split('\n'):
if line != '':
old_files.append(line.strip())
except:
pass
old_files = list(reversed(old_files))
# load unremoved conffiles list
unremoved_conffiles_f = open(Env.unremoved_conffiles(), 'r')
unremoved_conffiles = unremoved_conffiles_f.read().strip().split('\n')
unremoved_conffiles_f.close()
temp_dir = self.extracted_package_dir
# load files list from `files` directory of package
self.loaded_files = []
self.load_files(temp_dir + '/files', temp_dir + '/files')
self.loaded_files = [[target_path + f[0], f[1]]
for f in self.loaded_files]
# check file conflicts
all_installed_files = Pkg.get_all_installed_files_list()
for lf in self.loaded_files:
if not os.path.isdir(lf[1]):
for insf in all_installed_files:
if insf[0].split(':', 1)[0] != pkg.data['name']:
if lf[0] == insf[2]:
insf_pkg = Pkg.load_last(insf[0])
if insf_pkg:
insf[0] = insf[0] + ':' + insf_pkg.installed()
# check package is in `replaces` list
do_raise_error = True
replaces_list = pkg.get_replaces()
for rep in replaces_list:
if Pkg.check_state(rep):
tmp_parts = rep.split(' ')
tmp_parts = tmp_parts[0].split('>')
tmp_parts = tmp_parts[0].split('<')
tmp_parts = tmp_parts[0].split('=')
tmp_parts = tmp_parts[0]
if tmp_parts == insf[0].split(':', 1)[0]:
do_raise_error = False
if do_raise_error:
raise FileConflictError('package ' +
pkg.data['name'] +
':' +
pkg.data['version'] +
' and ' + insf[0] +
' both has file "' +
lf[0] + '"')
# copy loaded files
self.copied_files = []
for f in self.loaded_files:
if os.path.exists(Env.base_path(f[0])):
if os.path.isfile(Env.base_path(f[0])):
if ('f:' + f[0]) in old_files or ('cf:' +
f[0]) in old_files:
self.copy_once_file(f)
try:
old_files.pop(old_files.index(('f:' + f[0])))
except:
pass
else:
if f[0] in unremoved_conffiles:
self.copy_once_file(f)
unremoved_conffiles.pop(
unremoved_conffiles.index(f[0]))
else:
self.copy_once_file(f)
else:
if ('d:' + f[0]) in old_files or ('cd:' +
f[0]) in old_files:
if ('cd:' + f[0]) in old_files:
self.copied_files.append('cd:' + f[0])
old_files.pop(old_files.index(('cd:' + f[0])))
else:
self.copied_files.append('d:' + f[0])
old_files.pop(old_files.index(('d:' + f[0])))
else:
if f[0] in unremoved_conffiles:
self.copied_files.append('d:' + f[0])
unremoved_conffiles.pop(
unremoved_conffiles.index(f[0]))
else:
self.copy_once_file(f)
# delete not wanted old files
for item in old_files:
parts = item.strip().split(':', 1)
if parts[0] == 'cf' or parts[0] == 'cd':
pass
else:
if os.path.isfile(parts[1]):
os.remove(parts[1])
else:
try:
os.rmdir(parts[1])
except:
# directory is not emptyr
directory_not_empty_event(pkg, parts[1])
# write new unremoved conffiles list
unremoved_conffiles_f = open(Env.unremoved_conffiles(), 'w')
new_content = ''
for item in unremoved_conffiles:
new_content += item + '\n'
unremoved_conffiles_f.write(new_content)
unremoved_conffiles_f.close()
return self.copied_files
def install(self,
pkg: BaseArchive,
index_updater_events: dict,
installer_events: dict,
is_manual=True,
run_scripts=True,
target_path='',
keep_conffiles=False,
check_security_blacklist=True):
"""
Install .cati package
Args:
pkg (BaseArchive): the package archive object
index_updater_events (dict): events will be passed to `dotcati.ListUpdater.update_indexes()`
installer_events (dict): The events
- package_currently_install: gets the current installed version
- package_new_installs: gets package archive
- package_installed: will call after package installation
- dep_and_conflict_error: will run when there is depends or conflict error
- arch_error: will run when package arch is not sync with sys arch
is_manual (bool): installed package as manual or not (default is True means manual)
run_scripts (bool): run package install scripts or not (default is True)
target_path (str): where is the target path for installed files (will pass to `self.copy_files()`)
keep_conffiles (bool): stil keep config files if changed (default is True)
check_security_blacklist (bool): check package is in security blacklist or not
"""
self.conffiles = pkg.get_conffiles()
self.pkg = pkg
self.keep_conffiles = keep_conffiles
self.uncopied_conffiles = {}
# check package is in security blacklist
if check_security_blacklist:
self.check_security_blacklist(pkg)
# check package architecture
if not pkg.data['arch'] in SysArch.allowed_archs():
return installer_events['arch_error'](pkg)
# add package data to lists
self.add_package_to_lists(pkg, index_updater_events)
# check package dependencies and conflicts
try:
self.check_dep_and_conf(pkg)
except DependencyError as ex:
return installer_events['dep_and_conflict_error'](pkg, ex)
except ConflictError as ex:
return installer_events['dep_and_conflict_error'](pkg, ex)
# load old conffiles
self.old_conffiles = []
try:
f = open(
Env.installed_lists('/' + pkg.data['name'] + '/conffiles'),
'r')
content = f.read()
f.close()
tmp = content.strip().split('\n')
self.old_conffiles = [item.strip().split('@') for item in tmp]
except:
pass
# extract package in a temp place
temp_dir = Temp.make_dir()
os.rmdir(temp_dir)
try:
pkg.extractall(temp_dir)
except IsADirectoryError:
pass
self.extracted_package_dir = temp_dir
# install package
if Pkg.is_installed(pkg.data['name']):
installer_events['package_currently_installed'](
pkg, Pkg.installed_version(pkg.data['name']))
else:
installer_events['package_new_installs'](pkg)
if run_scripts:
self.run_script('ins-before')
copied_files = self.copy_files(pkg,
installer_events['directory_not_empty'],
target_path)
# set install configuration
if not os.path.isdir(Env.installed_lists('/' + pkg.data['name'])):
os.mkdir(Env.installed_lists('/' + pkg.data['name']))
f_ver = open(Env.installed_lists('/' + pkg.data['name'] + '/ver'), 'w')
f_ver.write(pkg.data['version']) # write installed version
f_ver.close()
# write copied files list
f_files = open(Env.installed_lists('/' + pkg.data['name'] + '/files'),
'w')
copied_files_str = ''
for copied_file in copied_files:
copied_files_str += copied_file + '\n'
f_files.write(copied_files_str.strip()) # write copied files
f_files.close()
# write conffiles list
f_conffiles = open(
Env.installed_lists('/' + pkg.data['name'] + '/conffiles'), 'w')
copied_conffiles_str = ''
for copied_conffile in copied_files:
if copied_conffile.split(':')[0] == 'cf':
try:
conffile_hash = self.uncopied_conffiles[
copied_conffile.split(':', 1)[-1]]
except:
conffile_hash = calc_file_sha256(
Env.base_path(copied_conffile.split(':', 1)[-1]))
copied_conffiles_str += conffile_hash + '@' + copied_conffile.split(
':', 1)[-1] + '\n'
f_conffiles.write(
copied_conffiles_str.strip()) # write copied conffiles
f_conffiles.close()
# copy `any` script
if os.path.isfile(self.extracted_package_dir + '/scripts/any'):
os.system('cp "' + self.extracted_package_dir + '/scripts/any' +
'" "' + Env.any_scripts('/' + pkg.data['name']) + '"')
# save static files list
static_files_list = pkg.get_static_files()
f_static_files = open(
Env.installed_lists('/' + pkg.data['name'] + '/staticfiles'), 'w')
static_files_str = ''
for copied_file in copied_files:
copied_file_path = copied_file.split(':', 1)[1]
if copied_file_path in static_files_list:
if os.path.isfile(Env.base_path('/' + copied_file_path)):
# calculate file sha256 sum
copied_file_sha256 = calc_file_sha256(
Env.base_path('/' + copied_file_path))
# add file to list
static_files_str += copied_file_sha256 + '@' + copied_file_path + '\n'
f_static_files.write(static_files_str.strip()) # write copied files
f_static_files.close()
f_installed_at = open(
Env.installed_lists('/' + pkg.data['name'] + '/installed_at'), 'w')
f_installed_at.write(str(time.time())) # write time (installed at)
f_installed_at.close()
if is_manual:
f_manual = open(
Env.installed_lists('/' + pkg.data['name'] + '/manual'), 'w')
f_manual.write('')
f_manual.close()
if run_scripts:
self.run_script('ins-after')
# copy remove scripts
if os.path.isfile(self.extracted_package_dir + '/scripts/rm-before'):
os.system('cp "' + self.extracted_package_dir +
'/scripts/rm-before' + '" "' +
Env.installed_lists('/' + pkg.data['name'] +
'/rm-before') + '"')
if os.path.isfile(self.extracted_package_dir + '/scripts/rm-after'):
os.system('cp "' + self.extracted_package_dir +
'/scripts/rm-after' + '" "' +
Env.installed_lists('/' + pkg.data['name'] +
'/rm-after') + '"')
# pop package from state
BaseTransaction.pop_state()
# call package installed event
installer_events['package_installed'](pkg)
