def __init__(
self, dist_version, log_file, log_level,
venv=None, git_repo='https://github.com/globz-eu/django_base.git', celery_pid='/var/run/django_base/celery',
fifo_dir='/tmp'
):
"""
Initializes parameters. Sets appropriate venv creation command and python version for distribution.
:param dist_version: distribution version
:param log_file: log file path
:param log_level: general log level
:param venv: venv path
:param git_repo: git repository URL
"""
CommandFileUtils.__init__(self, dist_version, log_file, log_level)
self.venv = venv
if self.venv:
self.pip = os.path.abspath(os.path.join(self.venv, 'bin/pip'))
if self.dist_version == '14.04':
self.venv_cmd = ['virtualenv', '-p', '/usr/bin/python3.4', self.venv]
self.python_version = 'python3.4'
elif self.dist_version == '16.04':
self.venv_cmd = ['pyvenv', self.venv]
self.python_version = 'python3.5'
else:
self.venv_cmd = None
self.python_version = None
self.git_repo = git_repo
p = re.compile('https://github.com/[\w\-]+/(\w+)\.git')
self.app_name = p.match(git_repo).group(1)
self.fifo_dir = fifo_dir
self.celery_pid = os.path.join(celery_pid, 'w1.pid')
def test_permissions_non_recursive(self):
"""
tests permissions assigns permissions recursively and writes to log
"""
test_permissions = [
[{'path': '/tmp/scripts_test/app_user/sites/app_name/source', 'dir_permissions': '500'}, 'dr-x------'],
[{'path': '/tmp/scripts_test/app_user/sites/app_name/source/app_name', 'dir_permissions': '770'}, 'drwxrwx---'],
[{'path': '/tmp/scripts_test/app_user/sites/app_name/source/app_name/file', 'file_permissions': '400'}, '-r--------'],
]
app_home_nested_file = os.path.join(self.app_home, 'app_name', 'file')
runlog = CommandFileUtils(self.dist_version, self.log_file, self.log_level)
for i in test_permissions:
os.makedirs(os.path.join(self.app_home, 'app_name'))
with open(app_home_nested_file, 'w') as file:
file.write('some text')
runlog.permissions(**i[0])
self.assertEqual(i[1], stat.filemode(os.stat(i[0]['path']).st_mode), stat.filemode(os.stat(i[0]['path']).st_mode))
if os.path.isdir(i[0]['path']):
self.log('INFO: changed permissions of %s to %s' % (i[0]['path'], i[0]['dir_permissions']))
elif os.path.isfile(i[0]['path']):
self.log('INFO: changed permissions of %s to %s' % (i[0]['path'], i[0]['file_permissions']))
os.chmod(self.app_home, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
for root, dirs, files in os.walk(self.app_home):
for name in dirs:
os.chmod(os.path.join(root, name), stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
shutil.rmtree(self.app_home)
def test_walktree_with_no_directory_function_args(self):
app_home_nested_file = os.path.join(self.app_home, 'app_name', 'file')
os.makedirs(os.path.join(self.app_home, 'app_name'))
with open(app_home_nested_file, 'w') as file:
file.write('some text')
runlog = CommandFileUtils(self.dist_version, self.log_file, self.log_level)
runlog.walktree(self.app_home, d_callback=runlog.write_to_log)
paths = ['/tmp/scripts_test/app_user/sites/app_name/source/app_name']
for p in paths:
self.log('DEBUG: %s' % p)
def test_walktree_with_no_file_function_args(self):
app_home_nested_file = os.path.join(self.test_dir, 'dir', 'file')
os.makedirs(os.path.join(self.test_dir, 'dir'))
with open(app_home_nested_file, 'w') as file:
file.write('some text')
runlog = CommandFileUtils(self.dist_version, self.log_file, self.log_level)
runlog.walktree(self.test_dir, f_callback=runlog.write_to_log)
paths = ['/tmp/scripts_test/dir/file']
for p in paths:
self.log('DEBUG: %s' % p)
def test_check_process(self):
"""
tests that check_process returns True when process is running and False otherwise
"""
runlog = CommandFileUtils(self.dist_version, self.log_file, self.log_level)
proc = runlog.check_process('python')
self.assertTrue(proc, '%s process is running: %s' % ('python', proc))
proc = runlog.check_process('SomeVeryUnlikelyProcessName')
self.assertFalse(proc, proc)
def test_run_command(self):
"""
tests run_and_log runs given command, exits on error and writes to log
"""
cmd = ['ls', '-la']
msg = 'successfully ran command'
runlog = CommandFileUtils(self.dist_version, self.log_file, self.log_level)
args = (cmd, msg)
runlog.run_command(*args)
self.log('INFO: %s' % msg)
def __init__(
self, dist_version, log_file, log_level, db_user, db_admin_user,
git_repo='https://github.com/globz-eu/django_base.git'
):
CommandFileUtils.__init__(self, dist_version, log_file, log_level)
self.git_repo = git_repo
p = re.compile('https://github.com/[\w\-]+/(\w+)\.git')
self.db_name = p.match(self.git_repo).group(1)
self.db_user = db_user
self.db_admin_user = db_admin_user
def test_check_pending_returns_correct_list(self):
"""
tests that check_pending only returns directories for pending or acceptance tests
"""
sample_dirs = ['/bla/pending_tests', '/bli/blo/acceptance_tests', '/bla/blup/some_other_dir']
expected_dirs = ['/bla/pending_tests', '/bli/blo/acceptance_tests']
cfu = CommandFileUtils(self.dist_version, self.log_file, self.log_level)
for s in sample_dirs:
cfu.check_pending(s)
self.assertEqual(expected_dirs, cfu.pending_dirs, cfu.pending_dirs)
def test_walktree_exits_when_it_encounters_permission_error(self):
"""
tests that walktree exits when it encounters a permission error while walking
"""
runlog = CommandFileUtils(self.dist_version, self.log_file, self.log_level)
try:
runlog.walktree('/etc', os.path.isfile, (), os.listdir, ())
except SystemExit as error:
self.assertEqual(1, error.code, '%s exited with: %s' % ('walktree', str(error)))
self.log('ERROR: Permission denied on: /etc/cups/ssl')
def test_walktree_with_no_file_function(self):
app_home_nested_file = os.path.join(self.app_home, 'app_name', 'file')
os.makedirs(os.path.join(self.app_home, 'app_name'))
with open(app_home_nested_file, 'w') as file:
file.write('some text')
runlog = CommandFileUtils(self.dist_version, self.log_file, self.log_level)
runlog.walktree(self.app_home, d_callback=runlog.write_to_log, d_args=('INFO', ))
paths = ['/tmp/scripts_test/app_user/sites/app_name/source/app_name']
for p in paths:
self.log('INFO: %s' % p)
self.log('ERROR:', test=False, regex=True)
def test_walktree(self):
app_home_nested_file = os.path.join(self.app_home, 'app_name', 'file')
os.makedirs(os.path.join(self.app_home, 'app_name'))
with open(app_home_nested_file, 'w') as file:
file.write('some text')
runlog = CommandFileUtils(self.dist_version, self.log_file, self.log_level)
runlog.walktree(self.app_home, runlog.write_to_log, ('INFO', ), runlog.write_to_log, ('INFO', ))
paths = ['/tmp/scripts_test/app_user/sites/app_name/source/app_name',
'/tmp/scripts_test/app_user/sites/app_name/source/app_name/file']
for p in paths:
self.log('INFO: %s' % p)
def test_run_command_exits_on_error(self):
"""
tests run_and_log runs given command, exits on error and writes to log
"""
cmd = ['ls', 'fjlskhgtioeb.bla']
msg = 'successfully ran command'
runlog = CommandFileUtils(self.dist_version, self.log_file, self.log_level)
args = (cmd, msg)
try:
runlog.run_command(*args)
self.fail('command did not raise error')
except SystemExit:
self.log('ERROR: ls fjlskhgtioeb.bla exited with exit code 2')
def __init__(self, dist_version='16.04', log_file='/tmp/install_from_url.log', log_level='INFO',
metadata_url=None, download_folder=DOWNLOAD_FOLDER):
"""
Initializes parameters.
:param dist_version: distribution version
:param log_file: log file path
:param log_level: log level
:param metadata_url: url for retrieving metadata
:param download_folder: folder to download package and metadata to
"""
CommandFileUtils.__init__(self, dist_version, log_file, log_level)
self.metadata_url = metadata_url
self.download_folder = download_folder
def clone_app_mock(app_home):
static_files = [
['static', 'static/static_file', 'static'],
['media', 'media/media_file', 'media'],
['', 'uwsgi_params', 'uwsgi_params'],
['static/base/site_down', 'static/base/site_down/index.html', 'index.html']
]
for static_dir, static_file_path, static_name in static_files:
os.makedirs(os.path.join(app_home, 'app_name', static_dir), exist_ok=True)
static_file_abs_path = os.path.join(app_home, 'app_name', static_file_path)
with open(static_file_abs_path, 'w') as static_file:
static_file.write('%s stuff\n' % static_name)
cfu = CommandFileUtils(DIST_VERSION, LOG_FILE, LOG_LEVEL)
cfu.write_to_log('successfully cloned %s to %s' % ('app_name', app_home), 'INFO')
def test_write_to_log(self):
"""
tests that write_to_log writes messages to log
"""
runlog = CommandFileUtils(self.dist_version, self.log_file, 'DEBUG')
msgs = [
['debug message', 'DEBUG'],
['info message', 'INFO'],
['warning message', 'WARNING'],
['error message', 'ERROR'],
['critical message', 'CRITICAL']
]
for m, ll in msgs:
runlog.write_to_log(m, ll)
self.log('%s: %s' % (ll, m))
def test_get_pending_dirs_returns_dirs_with_pending_tests(self):
"""
tests that get_pending_dirs returns a list of directory paths for pending tests
"""
os.makedirs(os.path.join(self.test_dir, 'dir', 'acceptance_tests'), exist_ok=True)
os.makedirs(os.path.join(self.test_dir, 'dir', 'dir', 'functional_tests', 'pending_tests'), exist_ok=True)
os.makedirs(os.path.join(self.test_dir, 'dir', 'base', 'unit_tests', 'pending_tests'), exist_ok=True)
cfu = CommandFileUtils(self.dist_version, self.log_file, self.log_level)
pending_dirs = cfu.get_pending_dirs(self.test_dir, 'dir')
expected_pending_dirs = [
os.path.join('.', 'acceptance_tests'),
os.path.join('.', 'base', 'unit_tests', 'pending_tests'),
os.path.join('.', 'dir', 'functional_tests', 'pending_tests'),
]
self.assertEqual(expected_pending_dirs, pending_dirs, pending_dirs)
def test_run_command_exits_on_error_and_does_not_log_when_log_error_is_false(self):
"""
tests run_and_log runs given command, exits on error and does not write to log when log_error is false
"""
cmd = ['ls', 'fjlskhgtioeb.bla']
msg = 'successfully ran command'
runlog = CommandFileUtils(self.dist_version, self.log_file, self.log_level)
args = (cmd, msg)
try:
runlog.run_command(*args, log_error=False)
self.fail('command did not raise error')
except SystemExit:
with open(self.log_file) as log:
log_content = log.readlines()
self.assertFalse('ERROR: ls fjlskhgtioeb.bla exited with exit code 2' in log_content, log_content)
def test_basic_functionality(self):
"""
tests basic logging functionality
"""
runlog = CommandFileUtils(self.dist_version, self.log_file, 'DEBUG')
msgs = [
['debug message', 'DEBUG'],
['info message', 'INFO'],
['warning message', 'WARNING'],
['error message', 'ERROR'],
['critical message', 'CRITICAL']
]
for m, ll in msgs:
runlog.write_to_log(m, ll)
for m, ll in msgs:
self.log('%s: %s' % (ll, m))
def test_level_functionality(self):
"""
tests logging functionality when log level is higher than DEBUG
"""
runlog = CommandFileUtils(self.dist_version, self.log_file, 'INFO')
msgs = [
['debug message', 'DEBUG'],
['info message', 'INFO'],
['warning message', 'WARNING'],
['error message', 'ERROR'],
['critical message', 'CRITICAL']
]
for m, ll in msgs:
runlog.write_to_log(m, ll)
for m, ll in msgs[1:]:
self.log('%s: %s' % (ll, m))
self.log('%s: %s' % (msgs[0][1], msgs[0][0]), test=False)
def test_own_manages_ownership(self, own_app_mock):
"""
tests that own manages ownership and writes to log.
"""
app_home_nested_file = os.path.join(self.app_home, 'app_name', 'file')
os.makedirs(os.path.join(self.app_home, 'app_name'))
with open(app_home_nested_file, 'w') as file:
file.write('some text')
user = '******'
group = 'app_user'
runlog = CommandFileUtils(self.dist_version, self.log_file, self.log_level)
runlog.own(self.app_home, user, group)
self.assertEqual([call(self.app_home, user, group)], own_app_mock.mock_calls, own_app_mock.mock_calls)
self.log('INFO: changed ownership of %s to %s:%s' % (self.app_home, user, user))
def test_permissions_recursive(self):
"""
tests permissions assigns permissions recursively and writes to log
"""
test_permissions = [
['500', '700', '-r-x------', 'drwx------'],
['400', '500', '-r--------', 'dr-x------'],
['550', '770', '-r-xr-x---', 'drwxrwx---'],
['440', '550', '-r--r-----', 'dr-xr-x---'],
['644', '755', '-rw-r--r--', 'drwxr-xr-x'],
['755', '755', '-rwxr-xr-x', 'drwxr-xr-x']
]
app_home_nested_file = os.path.join(self.app_home, 'app_name', 'file')
runlog = CommandFileUtils(self.dist_version, self.log_file, self.log_level)
for i in test_permissions:
os.makedirs(os.path.join(self.app_home, 'app_name'))
with open(app_home_nested_file, 'w') as file:
file.write('some text')
runlog.permissions(self.app_home, i[0], i[1], recursive=True)
app_home_files = []
app_home_dirs = []
for root, dirs, files in os.walk(self.app_home):
for name in files:
app_home_files.append(os.path.join(root, name))
for name in dirs:
app_home_dirs.append(os.path.join(root, name))
app_home_dirs.append(self.app_home)
for a in app_home_files:
self.assertEqual(i[2], stat.filemode(os.stat(a).st_mode), stat.filemode(os.stat(a).st_mode))
for a in app_home_dirs:
self.assertEqual(i[3], stat.filemode(os.stat(a).st_mode), stat.filemode(os.stat(a).st_mode))
self.log('INFO: changed permissions of %s files to %s and directories to %s' % (
self.app_home, i[0], i[1]
))
os.chmod(self.app_home, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
for root, dirs, files in os.walk(self.app_home):
for name in dirs:
os.chmod(os.path.join(root, name), stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
shutil.rmtree(self.app_home)
def test_write_to_log_adds_timestamp_to_message(self):
"""
tests that write_to_log adds the current time to messages
"""
runlog = CommandFileUtils(self.dist_version, self.log_file, 'DEBUG')
msgs = [
['debug message', 'DEBUG'],
['info message', 'INFO'],
['warning message', 'WARNING'],
['error message', 'ERROR'],
['critical message', 'CRITICAL']
]
now = datetime.datetime.utcnow()
for m, ll in msgs:
runlog.write_to_log(m, ll)
with open(self.log_file) as log:
log_list = [l[:19] for l in log][-5:]
for l in log_list:
self.assertEqual(now.strftime('%Y-%m-%d %H:%M:%S'), l, l)
def test_write_to_log_only_logs_messages_of_appropriate_log_level(self):
"""
tests that write_to_log only writes messages with appropriate log level to log
"""
runlog = CommandFileUtils(self.dist_version, self.log_file, 'ERROR')
msgs_log = [
['error message', 'ERROR'],
['CRITICAL message', 'CRITICAL']
]
msgs_no_log = [
['debug message', 'DEBUG'],
['info message', 'INFO'],
]
for m, ll in msgs_log:
runlog.write_to_log(m, ll)
self.log('%s: %s' % (ll, m))
for m, ll in msgs_no_log:
runlog.write_to_log(m, ll)
self.log('%s: %s' % (ll, m), test=False)
def test_write_to_log_exits_when_log_level_is_not_specified(self):
"""
tests that write_to_log uses default log level (DEBUG) when level is not specified and exits when log level is
invalid
"""
runlog = CommandFileUtils(self.dist_version, self.log_file, 'DEBUG')
msgs = [
['warning message', 'WARNING'],
['other warning message', 'WARNING'],
['debug message', 'IMPORTANT'],
['info message', 'INFO'],
['default info message', False],
]
for m, ll in msgs:
try:
runlog.write_to_log(m, ll)
self.log('%s: %s' % (ll, m))
except SystemExit as error:
self.assertEqual(1, error.code, '%s exited with: %s' % ('write_to_log', str(error)))
self.log('ERROR: log level "%s" is not specified or not valid' % ll)
def own_app_mock(path, owner, group):
cfu = CommandFileUtils(DIST_VERSION, LOG_FILE, LOG_LEVEL)
cfu.write_to_log('changed ownership of %s to %s:%s' % (path, owner, group), 'INFO')
def stop_celery_mock(app_home):
cfu = CommandFileUtils(DIST_VERSION, LOG_FILE, LOG_LEVEL)
cfu.write_to_log('stopped celery and beat', 'INFO')
if os.path.exists(os.path.join(CELERY_PID_PATH, 'w1.pid')):
os.remove(os.path.join(CELERY_PID_PATH, 'w1.pid'))
return 0