def unpack(ext, source, dest_path):
"""
Unpack the archive |source| to |dest_path|.
Note: |source| can be a file handle or a path.
|ext| contains the extension of the archive.
"""
close_source = False
try:
if isinstance(source, str):
source = open(source, 'rb')
close_source = True
if ext == '.tar.gz' or ext == '.tgz':
un_tar_directory(source, dest_path, 'gz')
elif ext == '.tar.bz2':
un_tar_directory(source, dest_path, 'bz2')
elif ext == '.bz2':
un_bz2_file(source, dest_path)
elif ext == '.gz':
with open(dest_path, 'wb') as f:
shutil.copyfileobj(un_gzip_stream(source), f)
elif ext == '.zip':
unzip_directory(source, dest_path)
else:
raise UsageError('Not an archive.')
except (tarfile.TarError, IOError):
raise UsageError('Invalid archive upload.')
finally:
if close_source:
source.close()
def test_tar_empty(self):
dir = tempfile.mkdtemp()
self.addCleanup(lambda: remove_path(dir))
temp_dir = tempfile.mkdtemp()
self.addCleanup(lambda: remove_path(temp_dir))
output_dir = os.path.join(temp_dir, 'output')
un_tar_directory(tar_gzip_directory(dir), output_dir, 'gz')
self.assertEqual(os.listdir(output_dir), [])
def test_tar_always_ignore(self):
temp_dir = tempfile.mkdtemp()
self.addCleanup(lambda: remove_path(temp_dir))
output_dir = os.path.join(temp_dir, 'output')
un_tar_directory(tar_gzip_directory(IGNORE_TEST_DIR), output_dir, 'gz')
output_dir_entries = os.listdir(output_dir)
self.assertNotIn('._ignored', output_dir_entries)
self.assertIn('dir', output_dir_entries)
self.assertNotIn('__MACOSX', output_dir_entries)
self.assertFalse(os.path.exists(os.path.join(output_dir, 'dir', '__MACOSX')))
self.assertFalse(os.path.exists(os.path.join(output_dir, 'dir', '._ignored2')))
def test_tar_exclude_ignore(self):
temp_dir = tempfile.mkdtemp()
self.addCleanup(lambda: remove_path(temp_dir))
output_dir = os.path.join(temp_dir, 'output')
un_tar_directory(
tar_gzip_directory(IGNORE_TEST_DIR, ignore_file='.tarignore'), output_dir, 'gz'
)
output_dir_entries = os.listdir(output_dir)
self.assertIn('not_ignored.txt', output_dir_entries)
self.assertIn('dir', output_dir_entries)
self.assertNotIn('ignored.txt', output_dir_entries)
self.assertNotIn('ignored_dir', output_dir_entries)
self.assertTrue(os.path.exists(os.path.join(output_dir, 'dir', 'not_ignored2.txt')))
self.assertFalse(os.path.exists(os.path.join(output_dir, 'dir', 'ignored2.txt')))
def test_tar_has_files(self):
temp_dir = tempfile.mkdtemp()
self.addCleanup(lambda: remove_path(temp_dir))
output_dir = os.path.join(temp_dir, 'output')
un_tar_directory(
tar_gzip_directory(FILES_DIR, False, ['f2'], ['f1', 'b.txt']), output_dir, 'gz'
)
output_dir_entries = os.listdir(output_dir)
self.assertIn('dir1', output_dir_entries)
self.assertIn('a.txt', output_dir_entries)
self.assertNotIn('b.txt', output_dir_entries)
self.assertTrue(os.path.exists(os.path.join(output_dir, 'dir1', 'f1')))
self.assertFalse(os.path.exists(os.path.join(output_dir, 'dir1', 'f2')))
self.assertTrue(os.path.islink(os.path.join(output_dir, 'a-symlink.txt')))
def unpack(ext, source, dest_path):
"""
Unpack the archive |source| to |dest_path|.
Note: |source| can be a file handle or a path.
|ext| contains the extension of the archive.
"""
if ext != '.zip':
close_source = False
try:
if isinstance(source, str):
source = open(source, 'rb')
close_source = True
if ext == '.tar.gz' or ext == '.tgz':
un_tar_directory(source, dest_path, 'gz')
elif ext == '.tar.bz2':
un_tar_directory(source, dest_path, 'bz2')
elif ext == '.bz2':
un_bz2_file(source, dest_path)
elif ext == '.gz':
with open(dest_path, 'wb') as f:
shutil.copyfileobj(un_gzip_stream(source), f)
else:
raise UsageError('Not an archive.')
except (tarfile.TarError, IOError):
raise UsageError('Invalid archive upload.')
finally:
if close_source:
source.close()
else:
delete_source = False
try:
# unzip doesn't accept input from standard input, so we have to save
# to a temporary file.
if not isinstance(source, str):
temp_path = dest_path + '.zip'
with open(temp_path, 'wb') as f:
shutil.copyfileobj(source, f)
source = temp_path
delete_source = True
exitcode = subprocess.call(
['unzip', '-q', source, '-d', dest_path])
if exitcode != 0:
raise UsageError('Invalid archive upload.')
finally:
if delete_source:
path_util.remove(source)
def _store_dependency(self, dependency_path, fileobj, target_type):
"""
Copy the dependency fileobj to its path on the local filesystem
Overwrite existing files by the same name if found
(may happen if filesystem modified outside the dependency manager,
for example during an update if the state gets reset but filesystem
doesn't get cleared)
"""
try:
if os.path.exists(dependency_path):
logger.info('Path %s already exists, overwriting', dependency_path)
if os.path.isdir(dependency_path):
shutil.rmtree(dependency_path)
else:
os.remove(dependency_path)
if target_type == 'directory':
un_tar_directory(fileobj, dependency_path, 'gz')
else:
with open(dependency_path, 'wb') as f:
logger.debug('copying file to %s', dependency_path)
shutil.copyfileobj(fileobj, f)
except Exception:
raise
def unarchive(self, *args, **kwargs):
return un_tar_directory(*args, **kwargs)