def add_files(self):
'''Get all the files present in a layer and store
them as a list of FileData objects'''
fs_path = rootfs.get_untar_dir(self.__tar_file)
hash_file = os.path.join(os.path.dirname(fs_path),
self.__fs_hash) + '.txt'
pattern = re.compile(r'([\w\-|]+)\s+(.+)')
with open(hash_file) as f:
content = f.readlines()
for line in content:
m = pattern.search(line)
if m:
# m.group(2) contains the file path
# m.group(1) contains the extattrs and checksum
file_data = FileData(os.path.basename(m.group(2)),
os.path.relpath(m.group(2), '.'))
# attrs_tuple contains (extattrs, '|', checksum)
attrs_tuple = m.group(1).rpartition('|')
file_data.set_checksum('sha256', attrs_tuple[2])
file_data.extattrs = attrs_tuple[0]
self.add_file(file_data)
def load_files_from_cache(layer):
'''Given a layer object, populate file level information'''
raw_file_list = cache.get_files(layer.fs_hash)
if raw_file_list:
logger.debug('Loading files from cache: layer \"%s\"',
layer.fs_hash[:10])
for file_dict in raw_file_list:
f = FileData(file_dict['name'], file_dict['path'])
f.fill(file_dict)
# collect file origins
if 'origins' in file_dict.keys():
for origin_dict in file_dict['origins']:
for notice in origin_dict['notices']:
f.origins.add_notice_to_origins(
origin_dict['origin_str'],
Notice(notice['message'], notice['level']))
layer.add_file(f)
else:
# if there are no files, generate them from the pre-calculated
# hash file
logger.debug('Reading files in filesystem...')
layer.add_files()
def _parse_hash_content(self, content):
"""This is an internal function to parse the content of the hash
and return a list of FileData objects
The content consists of lines of the form:
permissions|uid|gid|size|hard links| sha256sum filepath xattrs
where xattrs is the list of extended attributes for the file
The extended attributes start with a '# file' indicator, followed
by a list of key-value pairs separated by newlines. For now, we will
conserve the key-value pair list as they appear and separate
each one by a comma"""
file_list = []
# keep track of where we are on the list of files
index = 0
# loop through the content
while content:
line = content.pop(0)
if "# file" in line:
# collect the extended attributes
xattrlist = []
xattrline = content.pop(0)
while xattrline != '\n':
if 'selinux' not in xattrline:
xattrlist.append(xattrline.strip())
xattrline = content.pop(0)
# when we break out of the extended attributes loop
# we combine the results and update the FileData object
# existing in the previous index
file_list[index-1].extattrs = file_list[index-1].extattrs + \
" " + ','.join(xattrlist)
else:
# collect the regular attributes
file_info = line[:-1].split(' ')
file_data = FileData(os.path.basename(file_info[2]),
os.path.relpath(file_info[2], '.'))
file_data.set_checksum('sha256', file_info[1])
file_data.set_whiteout()
file_list.append(file_data)
index = index + 1
return file_list
def testFill(self):
file_dict = {
'name':
'zconf.h',
'path':
'/usr/include/zconf.h',
'checksum_type':
'sha256',
'checksum':
'77304005ceb5f0d03ad4c37eb8386a10866e'
'4ceeb204f7c3b6599834c7319541',
'extattrs':
'-rw-r--r-- 1 1000 1000 16262 Nov 13 17:57'
' /usr/include/zconf.h',
'checksums': [('SHA1', '12345abcde'),
('MD5', '1ff38cc592c4c5d0c8e3ca38be8f1eb1')]
}
f = FileData('zconf.h', '/usr/include/zconf.h')
f.fill(file_dict)
self.assertEqual(f.name, 'zconf.h')
self.assertEqual(f.path, '/usr/include/zconf.h')
self.assertEqual(f.checksum_type, 'sha256')
self.assertEqual(
f.checksum, '77304005ceb5f0d03ad4c37eb838'
'6a10866e4ceeb204f7c3b6599834c7319541')
self.assertEqual(
f.extattrs, '-rw-r--r-- 1 1000 1000 '
'16262 Nov 13 17:57 /usr/include/zconf.h')
self.assertEqual(f.checksums,
[('SHA1', '12345abcde'),
('MD5', '1ff38cc592c4c5d0c8e3ca38be8f1eb1')])
self.assertEqual(f.origins.origins[0].notices[1].message,
'No metadata for key: file_type')
self.assertEqual(f.origins.origins[0].notices[1].level, 'warning')
self.assertEqual(f.origins.origins[0].notices[0].message,
'No metadata for key: date')
self.assertEqual(f.origins.origins[0].notices[2].message,
'No metadata for key: version_control')
def testRemoveFile(self):
file1 = FileData('file1', 'path/to/file1')
self.layer.add_file(file1)
self.assertFalse(self.layer.remove_file('file1'))
self.assertTrue(self.layer.remove_file('path/to/file1'))
self.assertFalse(self.layer.remove_file('path/to/file1'))
def testExtAttrs(self):
file = FileData('test.txt', 'usr/abc/test.txt')
file.extattrs = '-rw-r--r--|1000|1000|19|1'
self.assertEqual(file.extattrs, '-rw-r--r--|1000|1000|19|1')
def setUp(self):
self.afile = FileData('afile', 'path/to/afile')
self.afile.licenses = ['MIT', 'GPL']
def testSetWhiteout(self):
whiteout = FileData('.wh.os-release', 'usr/lib/.wh.os-release')
whiteout.set_whiteout()
self.assertTrue(whiteout.is_whiteout)
self.afile.set_whiteout()
self.assertFalse(self.afile.is_whiteout)
def setUp(self):
self.command1 = Command("yum install nfs-utils")
self.command2 = Command("yum remove nfs-utils")
self.image = TestImage('5678efgh')
self.file = FileData('README.txt', '/home/test')
cache.cache = {}
