def setUp(self):
self.command1 = Command("yum install nfs-utils")
self.command2 = Command("yum remove nfs-utils")
self.image = TestImage('5678efgh')
self.file = FileData('README.txt', '/home/test')
cache.cache = {}
self.test_dockerfile = 'tests/dockerfiles/buildpack_deps_jessie_curl'
def testConsolidateCommandsWithSameCommands(self):
command = Command("yum install nfs-utils")
commands_list = fltr.consolidate_commands([self.command1, command])
self.assertEqual(len(commands_list), 1)
self.assertEqual(len(commands_list[0].words), 2)
command = Command("yum remove nfs-utils")
commands_list = fltr.consolidate_commands([self.command2, command])
self.assertEqual(len(commands_list), 1)
self.assertEqual(len(commands_list[0].words), 2)
def get_shell_commands(shell_command_line):
'''Given a shell command line, get a list of Command objects'''
statements = general.split_command(shell_command_line)
command_list = []
# traverse the statements, pick out the loop and commands.
for stat in statements:
if 'command' in stat:
command_list.append(Command(stat['command']))
elif 'loop' in stat:
loop_stat = stat['loop']['loop_statements']
for st in loop_stat:
if 'command' in st:
command_list.append(Command(st['command']))
return command_list
def get_shell_commands(shell_command_line):
'''Given a shell command line, get a list of Command objects'''
comm_list = general.split_command(shell_command_line)
cleaned_list = []
for comm in comm_list:
cleaned_list.append(Command(general.clean_command(comm)))
return cleaned_list
def get_shell_commands(shell_command_line):
'''Given a shell command line, get a list of Command objects'''
comm_list = shell_command_line.split('&&')
cleaned_list = []
for comm in comm_list:
cleaned_list.append(Command(comm.strip()))
return cleaned_list
def get_shell_commands(shell_command_line):
'''Given a shell command line, get a list of Command objects and report on
branch statements'''
statements = general.split_command(shell_command_line)
command_list = []
branch_report = ''
# traverse the statements, pick out the loop and commands.
for stat in statements:
if 'command' in stat:
command_list.append(Command(stat['command']))
elif 'loop' in stat:
loop_stat = stat['loop']['loop_statements']
for st in loop_stat:
if 'command' in st:
command_list.append(Command(st['command']))
elif 'branch' in stat:
branch_report = branch_report + '\n'.join(stat['content']) + '\n\n'
if branch_report:
# add prefix
branch_report = '\nNon-deterministic branching statement: \n' + \
branch_report
return command_list, branch_report
def setUp(self):
self.command1 = Command("yum install nfs-utils")
self.command2 = Command("yum remove nfs-utils")
class TestAnalyzeDefaultFilter(unittest.TestCase):
def setUp(self):
self.command1 = Command("yum install nfs-utils")
self.command2 = Command("yum remove nfs-utils")
def tearDown(self):
del self.command1
del self.command2
def testFilterInstallCommands(self):
commands, _ = fltr.filter_install_commands("yum install")
self.assertEqual(len(commands), 1)
commands, _ = fltr.filter_install_commands("yum remove")
self.assertEqual(len(commands), 1)
# Negative Scenarios
commands, _ = fltr.filter_install_commands("yum clean")
self.assertEqual(len(commands), 0)
commands, _ = fltr.filter_install_commands("yum")
self.assertEqual(len(commands), 0)
def testConsolidateCommandsWithDifferentCommands(self):
commands_list = fltr.consolidate_commands(
[self.command1, self.command2])
self.assertEqual(len(commands_list), 1)
self.assertEqual(len(commands_list[0].words), 3)
def testConsolidateCommandsWithSameCommands(self):
command = Command("yum install nfs-utils")
commands_list = fltr.consolidate_commands([self.command1, command])
self.assertEqual(len(commands_list), 1)
self.assertEqual(len(commands_list[0].words), 2)
command = Command("yum remove nfs-utils")
commands_list = fltr.consolidate_commands([self.command2, command])
self.assertEqual(len(commands_list), 1)
self.assertEqual(len(commands_list[0].words), 2)
def testRemoveIgnoredCommandsWithIgnoreFlag(self):
self.command1.set_ignore()
_, c = fltr.remove_ignored_commands([self.command1])
self.assertEqual(len(c), 0)
def testRemoveIgnoredCommandsWithoutIgnoreFlag(self):
# Negative Scenarios
_, c = fltr.remove_ignored_commands([self.command1])
self.assertEqual(len(c), 1)
def testRemoveUnrecognizedCommandsWithoutFlag(self):
_, c = fltr.remove_unrecognized_commands([self.command1])
self.assertEqual(len(c), 0)
def testRemoveUnrecognizedCommandsWithFlag(self):
# Negative Scenarios
self.command1.set_install()
_, c = fltr.remove_unrecognized_commands([self.command1])
self.assertEqual(len(c), 1)
def testGetInstalledPackageNamesWithInstallFlag(self):
self.command1.set_install()
self.assertGreater(
len(fltr.get_installed_package_names(self.command1)), 0)
def testGetInstalledPackageNamesWithRemoveFlag(self):
# Negative Scenarios
self.command2.set_remove()
self.assertEqual(
len(fltr.get_installed_package_names(self.command2)), 0)
class TestAnalyzeCommon(unittest.TestCase):
def setUp(self):
self.command1 = Command("yum install nfs-utils")
self.command2 = Command("yum remove nfs-utils")
self.image = TestImage('5678efgh')
self.file = FileData('README.txt', '/home/test')
cache.cache = {}
self.test_dockerfile = 'tests/dockerfiles/buildpack_deps_jessie_curl'
def tearDown(self):
del self.image
del self.command1
del self.command2
cache.cache = {}
del self.test_dockerfile
def testGetShellCommands(self):
command = common.get_shell_commands("yum install nfs-utils")
self.assertEqual(type(command), list)
self.assertEqual(len(command), 1)
self.assertEqual(command[0].options, self.command1.options)
def testLoadFromCache(self):
'''Given a layer object, populate the given layer in case the cache isn't empty'''
self.image.load_image()
# No cache
self.assertFalse(
common.load_from_cache(self.image.layers[0], redo=True))
# Cache populated
layer = self.image.layers[0]
cache.cache = {
layer.fs_hash: {
'files_analyzed': False,
'os_guess': 'Ubuntu',
'pkg_format': 'tar',
'files': [{
'name': 'README.md',
'path': '/home/test/projectsX'
}],
'extension_info': {},
'packages': [{
'name': 'README'
}]
}
}
self.assertTrue(
common.load_from_cache(self.image.layers[0], redo=False))
def testLoadFilesFromCache(self):
'''Given a layer object, populate the files of given layer in case the cache isn't empty'''
self.image.load_image()
# not loaded in cache
layer = self.image.layers[0]
layer.add_files = Mock(return_value=None)
self.assertFalse(common.load_files_from_cache(layer))
self.assertTrue(layer.add_files.called)
# Empty files
layer = self.image.layers[0]
cache.cache = {
layer.fs_hash: {
'files_analyzed': False,
'files': [],
'packages': [{
'name': 'README.md',
'path': '/home/test/projectsX'
}]
}
}
layer.add_files = Mock(return_value=None)
self.assertFalse(common.load_files_from_cache(layer))
self.assertTrue(layer.add_files.called)
# With no empty files
layer.add_files = Mock(return_value=None)
cache.cache[layer.fs_hash]['files'].append({
'name':
'README.md',
'path':
'/home/test/projectsX',
'origins': [{
'origin_str':
'security issue',
'notices': [{
'message': 'Missing security policy',
'level': 'info'
}]
}]
})
self.assertEqual(common.load_files_from_cache(layer), None)
self.assertFalse(layer.add_files.called)
def testLoadPackagesFromCache(self):
'''Given a layer object, populate the packages of given layer in case the cache isn't empty'''
self.image.load_image()
# not loaded in cache
layer = self.image.layers[0]
layer.add_package = Mock(return_value=None)
self.assertFalse(common.load_packages_from_cache(layer))
self.assertFalse(layer.add_package.called)
# Empty files
layer = self.image.layers[0]
cache.cache = {
layer.fs_hash: {
'files_analyzed': False,
'files': [],
'packages': []
}
}
layer.add_package = Mock(return_value=None)
self.assertFalse(common.load_packages_from_cache(layer))
self.assertFalse(layer.add_package.called)
# With no empty files
layer.add_package = Mock(return_value=None)
cache.cache[layer.fs_hash]['packages'].append({
'name':
'README.md',
'origins': [{
'origin_str':
'security issue',
'notices': [{
'message': 'Missing security policy',
'level': 'info'
}]
}]
})
self.assertTrue(common.load_packages_from_cache(layer))
self.assertTrue(layer.add_package.called)
def testLoadNoticesFromCache(self):
'''Given a layer object, populate the notices messages of given layer in case the cache isn't empty'''
origin_str = 'security issue'
self.image.load_image()
# not loaded in cache
layer = self.image.layers[0]
cache.cache = {layer.fs_hash: {'origins': []}}
layer.origins.add_notice_origin = Mock(return_value=None)
layer.origins.add_notice_to_origins = Mock(return_value=None)
common.load_notices_from_cache(layer)
self.assertFalse(layer.origins.add_notice_origin.called)
self.assertFalse(layer.origins.add_notice_to_origins.called)
# With no empty
layer.origins.add_notice_origin = Mock(return_value=None)
layer.origins.add_notice_to_origins = Mock(return_value=None)
cache.cache[layer.fs_hash]['origins'].append({
'origin_str':
origin_str,
'notices': [{
'message': 'Missing security policy',
'level': 'info'
}]
})
common.load_notices_from_cache(layer)
layer.origins.add_notice_origin.assert_called_with(origin_str)
self.assertTrue(layer.origins.add_notice_to_origins.called)
def testSaveToCache(self):
'''Given a image object, check the image layers are cached'''
self.image.load_image()
layer = self.image.layers[0]
cache.add_layer = Mock()
common.save_to_cache(self.image)
cache.add_layer.assert_called_once_with(layer)
def testUpdateMasterListWithoutPackages(self):
self.image.load_image()
layer = self.image.layers[0]
master_list = list()
common.update_master_list(master_list, layer)
self.assertEqual(len(master_list), len(layer.packages))
def testUpdateMasterListWithPackages(self):
self.image.load_image()
layer = self.image.layers[0]
master_list = list()
older_master_list = list()
for pkg in layer.packages:
master_list.append(pkg)
older_master_list.append(pkg)
common.update_master_list(master_list, layer)
self.assertEqual(len(master_list), len(older_master_list))
self.assertEqual(len(layer.packages), 0)
for old_pkg in older_master_list:
exists = False
for pkg in master_list:
if old_pkg.is_equal(pkg):
exists = True
break
self.assertTrue(exists)
def testFilterInstallCommands(self):
commands, _ = common.filter_install_commands("yum install")
self.assertEqual(len(commands), 1)
commands, _ = common.filter_install_commands("yum remove")
self.assertEqual(len(commands), 1)
# Negative Scenarios
commands, _ = common.filter_install_commands("yum clean")
self.assertEqual(len(commands), 0)
commands, _ = common.filter_install_commands("yum")
self.assertEqual(len(commands), 0)
def testConsolidateCommandsWithDifferentCommands(self):
commands_list = common.consolidate_commands(
[self.command1, self.command2])
self.assertEqual(len(commands_list), 1)
self.assertEqual(len(commands_list[0].words), 3)
def testConsolidateCommandsWithSameCommands(self):
command = Command("yum install nfs-utils")
commands_list = common.consolidate_commands([self.command1, command])
self.assertEqual(len(commands_list), 1)
self.assertEqual(len(commands_list[0].words), 2)
command = Command("yum remove nfs-utils")
commands_list = common.consolidate_commands([self.command2, command])
self.assertEqual(len(commands_list), 1)
self.assertEqual(len(commands_list[0].words), 2)
def testRemoveIgnoredCommandsWithIgnoreFlag(self):
self.command1.set_ignore()
_, c = common.remove_ignored_commands([self.command1])
self.assertEqual(len(c), 0)
def testRemoveIgnoredCommandsWithoutIgnoreFlag(self):
# Negative Scenarios
_, c = common.remove_ignored_commands([self.command1])
self.assertEqual(len(c), 1)
def testRemoveUnrecognizedCommandsWithoutFlag(self):
_, c = common.remove_unrecognized_commands([self.command1])
self.assertEqual(len(c), 0)
def testRemoveUnrecognizedCommandsWithFlag(self):
# Negative Scenarios
self.command1.set_install()
_, c = common.remove_unrecognized_commands([self.command1])
self.assertEqual(len(c), 1)
def testGetInstalledPackageNamesWithInstallFlag(self):
self.command1.set_install()
self.assertGreater(
len(common.get_installed_package_names(self.command1)), 0)
def testGetInstalledPackageNamesWithRemoveFlag(self):
# Negative Scenarios
self.command2.set_remove()
self.assertEqual(
len(common.get_installed_package_names(self.command2)), 0)
def testGetGitURL(self):
''' we check the url to be the following form:
github.com/<username>/tern'''
url_list = common.get_git_url(self.test_dockerfile)
check_num = len(url_list)
pass_num = 0
git_username_reg = r'([a-zA-Z\d_-]{0,38})'
pattern = re.compile(r'github.com/' + git_username_reg + r'/tern')
for url in url_list:
if pattern.match(url):
pass_num += 1
self.assertEqual(pass_num, check_num)
def setUp(self):
self.command1 = Command("yum install nfs-utils")
self.image = TestImage('5678efgh')
cache.cache = {}
self.test_dockerfile = 'tests/dockerfiles/buildpack_deps_jessie_curl'
def setUp(self):
self.install = Command('apt-get install -y git')
self.untar = Command('tar -x -C file tarfile.tar')
self.download = Command('wget url')
self.remove = Command('apt-get purge git')
self.install2 = Command('apt-get install -y ca-certificates')
class TestClassCommand(unittest.TestCase):
def setUp(self):
self.install = Command('apt-get install -y git')
self.untar = Command('tar -x -C file tarfile.tar')
self.download = Command('wget url')
self.remove = Command('apt-get purge git')
self.install2 = Command('apt-get install -y ca-certificates')
def tearDown(self):
del self.install
del self.untar
del self.download
del self.remove
del self.install2
def testInstance(self):
self.assertEqual(self.install.shell_command, 'apt-get install -y git')
self.assertEqual(self.install.name, 'apt-get')
# at this point the parser don't know that install is a subcommand
self.assertFalse(self.install.subcommand)
self.assertEqual(len(self.install.options), 1)
self.assertEqual(self.install.options[0][0], '-y')
# git isn't an option argument but it should still be in the option
# tuple as it comes after -y
self.assertEqual(self.install.options[0][1], 'git')
self.assertEqual(len(self.install.words), 2)
self.assertEqual(self.install.words[0], 'install')
self.assertEqual(self.install.words[1], 'git')
self.assertEqual(self.install.flags, 0)
self.assertFalse(self.install.is_set())
self.assertEqual(self.untar.shell_command,
'tar -x -C file tarfile.tar')
self.assertEqual(self.untar.name, 'tar')
self.assertFalse(self.untar.subcommand)
self.assertEqual(len(self.untar.options), 2)
self.assertFalse(self.untar.options[0][1])
self.assertEqual(self.untar.options[1][1], 'file')
# there are 2 words - file and tarfile.tar
self.assertEqual(len(self.untar.words), 2)
self.assertEqual(self.untar.words[0], 'file')
self.assertEqual(self.untar.words[1], 'tarfile.tar')
self.assertEqual(self.untar.flags, 0)
self.assertFalse(self.untar.is_set())
self.assertEqual(self.download.name, 'wget')
self.assertFalse(self.download.subcommand)
self.assertFalse(self.download.options)
self.assertEqual(len(self.download.words), 1)
self.assertEqual(self.download.words[0], 'url')
self.assertEqual(self.download.flags, 0)
self.assertFalse(self.download.is_set())
def testReassignWord(self):
# install is a subcommand
self.assertTrue(self.install.reassign_word('install', 'subcommand'))
self.assertFalse(self.install.reassign_word('install', 'subcommand'))
# 'file' is an option argument
self.assertTrue(self.untar.reassign_word('file', 'option_arg'))
self.assertFalse(self.untar.reassign_word('file', 'option_arg'))
# wget has no subcommands
self.assertFalse(self.download.reassign_word('safe', 'subcommand'))
def testGetOptionArgument(self):
# in the case of the install command -y has no options but if it
# did it should return 'git'
self.assertEqual(self.install.get_option_argument('-y'), 'git')
# for the tar command '-C' has the argument 'file'
self.assertEqual(self.untar.get_option_argument('-C'), 'file')
# for the wget command there are no options so this should
# return None
self.assertEqual(self.download.get_option_argument('-f'), None)
def testFlags(self):
# move install subcommand, then set the flag, then check to
# see if the command is an install command
self.install.reassign_word('install', 'subcommand')
self.install.set_install()
self.assertTrue(self.install.is_set())
self.assertTrue(self.install.is_install())
# ignore wget
self.download.set_ignore()
self.assertTrue(self.download.is_set())
self.assertTrue(self.download.is_ignore())
# set apt-get purge as a remove command
self.remove.set_remove()
self.assertTrue(self.remove.is_set())
self.assertTrue(self.remove.is_remove())
def testMerge(self):
self.install.reassign_word('install', 'subcommand')
self.install.set_install()
self.install2.reassign_word('install', 'subcommand')
self.install2.set_install()
self.remove.reassign_word('purge', 'subcommand')
self.remove.set_remove()
# merge the second install into the first install
self.assertTrue(self.install.merge(self.install2))
self.assertTrue(self.install.words, ['git', 'ca-certificates'])
# merge the remove command
self.assertTrue(self.install.merge(self.remove))
self.assertTrue(self.install.words, ['ca-certificates'])
# try to merge the download command
self.assertFalse(self.install.merge(self.download))
# try to merge an ignored command
self.download.set_ignore()
self.assertFalse(self.install.merge(self.download))
# try to merge some other object
with self.assertRaises(TypeError):
self.install.merge('test')
def setUp(self):
self.command1 = Command("yum install nfs-utils")
self.command2 = Command("yum remove nfs-utils")
self.image = TestImage('5678efgh')
self.file = FileData('README.txt', '/home/test')
cache.cache = {}
def setUp(self):
self.install = Command('apt-get install -y git')
self.untar = Command('tar -x -C file tarfile.tar')
self.download = Command('wget url')
self.remove = Command('apt-get purge git')
self.install2 = Command('apt-get install -y ca-certificates')
self.pinned1 = Command('apt-get install python3 zlib1g-dev=4.52.1')
self.pinned2 = Command('apt-get install ca-certificates:ppc64=2018i')
self.pinned3 = Command('yum install libncurses5-dev syslog-ng-1-2.3')
def setUp(self):
self.command1 = Command("yum install nfs-utils")
self.command2 = Command("yum remove nfs-utils")
self.image = TestImage('5678efgh')
class TestAnalyzeCommon(unittest.TestCase):
def setUp(self):
self.command1 = Command("yum install nfs-utils")
self.command2 = Command("yum remove nfs-utils")
self.image = TestImage('5678efgh')
def tearDown(self):
del self.image
del self.command1
del self.command2
def testGetShellCommands(self):
command = common.get_shell_commands("yum install nfs-utils")
self.assertEqual(type(command), list)
self.assertEqual(len(command), 1)
self.assertEqual(command[0].options, self.command1.options)
def testUpdateMasterListWithoutPackages(self):
self.image.load_image()
layer = self.image.layers[0]
master_list = list()
common.update_master_list(master_list, layer)
self.assertEqual(len(master_list), len(layer.packages))
def testUpdateMasterListWithPackages(self):
self.image.load_image()
layer = self.image.layers[0]
master_list = list()
older_master_list = list()
for pkg in layer.packages:
master_list.append(pkg)
older_master_list.append(pkg)
common.update_master_list(master_list, layer)
self.assertEqual(len(master_list), len(older_master_list))
self.assertEqual(len(layer.packages), 0)
for old_pkg in older_master_list:
exists = False
for pkg in master_list:
if old_pkg.is_equal(pkg):
exists = True
break
self.assertTrue(exists)
def testFilterInstallCommands(self):
commands, _ = common.filter_install_commands("yum install")
self.assertEqual(len(commands), 1)
commands, _ = common.filter_install_commands("yum remove")
self.assertEqual(len(commands), 1)
# Negative Scenarios
commands, _ = common.filter_install_commands("yum clean")
self.assertEqual(len(commands), 0)
commands, _ = common.filter_install_commands("yum")
self.assertEqual(len(commands), 0)
def testConsolidateCommandsWithDifferentCommands(self):
commands_list = common.consolidate_commands(
[self.command1, self.command2])
self.assertEqual(len(commands_list), 1)
self.assertEqual(len(commands_list[0].words), 3)
def testConsolidateCommandsWithSameCommands(self):
command = Command("yum install nfs-utils")
commands_list = common.consolidate_commands([self.command1, command])
self.assertEqual(len(commands_list), 1)
self.assertEqual(len(commands_list[0].words), 2)
command = Command("yum remove nfs-utils")
commands_list = common.consolidate_commands([self.command2, command])
self.assertEqual(len(commands_list), 1)
self.assertEqual(len(commands_list[0].words), 2)
def testRemoveIgnoredCommandsWithIgnoreFlag(self):
self.command1.set_ignore()
_, c = common.remove_ignored_commands([self.command1])
self.assertEqual(len(c), 0)
def testRemoveIgnoredCommandsWithoutIgnoreFlag(self):
# Negative Scenarios
_, c = common.remove_ignored_commands([self.command1])
self.assertEqual(len(c), 1)
def testRemoveUnrecognizedCommandsWithoutFlag(self):
_, c = common.remove_unrecognized_commands([self.command1])
self.assertEqual(len(c), 0)
def testRemoveUnrecognizedCommandsWithFlag(self):
# Negative Scenarios
self.command1.set_install()
_, c = common.remove_unrecognized_commands([self.command1])
self.assertEqual(len(c), 1)
def testGetInstalledPackageNamesWithInstallFlag(self):
self.command1.set_install()
self.assertGreater(
len(common.get_installed_package_names(self.command1)), 0)
def testGetInstalledPackageNamesWithRemoveFlag(self):
# Negative Scenarios
self.command2.set_remove()
self.assertEqual(
len(common.get_installed_package_names(self.command2)), 0)