def test_github_14(self):
# decompress content without filename
fp = self._open_file(os.path.join(ROOT, 'data', 'github_14.7z'), 'rb')
archive = Archive7z(fp)
self.assertEqual(archive.getnames(), ['github_14'])
cf = archive.getmember('github_14')
self.assertNotEqual(cf, None)
data = cf.read()
self.assertEqual(len(data), cf.uncompressed)
self.assertEqual(data, bytes('Hello GitHub issue #14.\n', 'ascii'))
# accessing by name returns an arbitrary compressed streams
# if both don't have a name in the archive
fp = self._open_file(os.path.join(ROOT, 'data', 'github_14_multi.7z'), 'rb')
archive = Archive7z(fp)
self.assertEqual(archive.getnames(), ['github_14_multi', 'github_14_multi'])
cf = archive.getmember('github_14_multi')
self.assertNotEqual(cf, None)
data = cf.read()
self.assertEqual(len(data), cf.uncompressed)
self.assertTrue(data in (bytes('Hello GitHub issue #14 1/2.\n', 'ascii'), bytes('Hello GitHub issue #14 2/2.\n', 'ascii')))
# accessing by index returns both values
cf = archive.getmember(0)
self.assertNotEqual(cf, None)
data = cf.read()
self.assertEqual(len(data), cf.uncompressed)
self.assertEqual(data, bytes('Hello GitHub issue #14 1/2.\n', 'ascii'))
cf = archive.getmember(1)
self.assertNotEqual(cf, None)
data = cf.read()
self.assertEqual(len(data), cf.uncompressed)
self.assertEqual(data, bytes('Hello GitHub issue #14 2/2.\n', 'ascii'))
def test_github_53(self):
# test loading file submitted by @VinylChloride
fp = self._open_file(os.path.join(ROOT, 'data', 'github_53.7z'), 'rb')
archive = Archive7z(fp, password='******')
self.assertEqual(sorted(archive.getnames()),
['test/test/archive7zip.py', 'test/test/mainPrg.py'])
self._test_decode_all(archive)
def un7zip_api_response(content_7z: bytes) -> Optional[bytes]:
try:
buffer = BytesIO(content_7z)
archive = Archive7z(buffer, password=NAPI_ARCHIVE_PASSWORD)
return archive.getmember(0).read()
except ArchiveError:
return None
def _test_archive(self, filename):
fp = self._open_file(os.path.join(ROOT, 'data', filename), 'rb')
archive = Archive7z(fp)
self.assertEqual(sorted(archive.getnames()),
['test/test2.txt', 'test1.txt'])
self.assertEqual(archive.getmember('test2.txt'), None)
cf = archive.getmember('test1.txt')
self.assertTrue(cf.checkcrc())
self.assertEqual(cf.lastwritetime // 10000000, 12786932628)
self.assertEqual(cf.lastwritetime.as_datetime().replace(microsecond=0), \
datetime(2006, 3, 15, 21, 43, 48, 0, UTC))
self.assertEqual(cf.read(),
bytes('This file is located in the root.', 'ascii'))
cf.reset()
self.assertEqual(cf.read(),
bytes('This file is located in the root.', 'ascii'))
cf = archive.getmember('test/test2.txt')
self.assertTrue(cf.checkcrc())
self.assertEqual(cf.lastwritetime // 10000000, 12786932616)
self.assertEqual(cf.lastwritetime.as_datetime().replace(microsecond=0), \
datetime(2006, 3, 15, 21, 43, 36, 0, UTC))
self.assertEqual(cf.read(),
bytes('This file is located in a folder.', 'ascii'))
cf.reset()
self.assertEqual(cf.read(),
bytes('This file is located in a folder.', 'ascii'))
def sevenZipTest(self):
try:
Archive7z(open(self.path)).getnames()
except:
return False
else:
return True
def extractFromFileAndCrc(self, crc, dest):
if not gotLib7z:
writeError("Sorry, You cannot use 7z file")
return False
writeInfo("Extract 7Z file")
fp = open(self.file, 'rb')
archive = Archive7z(fp)
for name in archive.getnames():
fd = archive.getmember(name)
crcFd = getCRCFromBuffer(fd.read())
if crc == crcFd:
writeInfo("On extracte un 7z %s" % file)
fd.reset()
data = fd.read()
nameDest = os.path.basename(name)
writeInfo("On extract du 7z " + nameDest)
fdest = NamedTemporaryFile()
#fdest = open("tmp" + "/" + nameDest, 'w+b', os.O_SYNC)
fdest.write(data)
fdest.flush()
#fdest.close()
print "Utilisation du fichier temporaire %s" % fdest.name
#truncateRomFile("tmp" + "/" + nameDest, dest + "/" + nameDest)
truncateRomFile(fdest.name, dest + "/" + nameDest)
fdest.close()
#os.remove("tmp" + "/" + nameDest)
fp.close()
writeInfo("Ecriture Finie")
def test_encrypted_wong_password(self):
# test loading of encrypted files with wrong password
fp = open(os.path.join(ROOT, 'data', 'encrypted.7z'), 'rb')
archive = Archive7z(fp, password='******')
filenames = archive.getnames()
for filename in filenames:
cf = archive.getmember(filename)
self.failUnlessRaises(WrongPasswordError, cf.read)
def test_encrypted_password(self):
# test loading of encrypted files with correct password
fp = open(os.path.join(ROOT, 'data', 'encrypted.7z'), 'rb')
archive = Archive7z(fp, password='******')
filenames = archive.getnames()
for filename in filenames:
cf = archive.getmember(filename)
self.failUnlessEqual(len(cf.read()), cf.uncompressed)
def test_bugzilla_16(self):
# sample file for bugzilla #16
fp = open(os.path.join(ROOT, 'data', 'bugzilla_16.7z'), 'rb')
archive = Archive7z(fp)
filenames = archive.getnames()
for filename in filenames:
cf = archive.getmember(filename)
self.failUnlessEqual(len(cf.read()), cf.uncompressed)
def test_github_43_provided(self):
# test loading file submitted by @mikenye
fp = self._open_file(os.path.join(ROOT, 'data', 'test-issue-43.7z'),
'rb')
archive = Archive7z(fp)
self.assertEqual(sorted(archive.getnames()), ['blah.txt'] +
['blah%d.txt' % x for x in xrange(2, 10)])
self._test_decode_all(archive)
def _access_7z(path):
try:
fp = open(path, 'rb')
archive = Archive7z(fp)
except:
archive = False
raise
return archive
def test_encrypted_no_password(self):
# test loading of encrypted files without password (can read archived filenames)
fp = self._open_file(os.path.join(ROOT, 'data', 'encrypted.7z'), 'rb')
archive = Archive7z(fp)
filenames = archive.getnames()
self.assertEqual(sorted(archive.getnames()), sorted(['test1.txt', 'test/test2.txt']))
self.assertRaises(NoPasswordGivenError, archive.getmember('test1.txt').read)
self.assertRaises(NoPasswordGivenError, archive.getmember('test/test2.txt').read)
def test_github_41(self):
fp = self._open_file(os.path.join(ROOT, 'data', 'github_41.7z'), 'rb')
archive = Archive7z(fp)
# Also only an empty file (which gets filtered).
self.assertEqual(archive.getnames(), [])
self.assertEqual(archive.header.files.numfiles, 1)
self.assertEqual([x['filename'] for x in archive.header.files.files], [u'12313213.txt'])
cf = archive.getmember('12313213.txt')
self.assertEqual(cf, None)
def _test_umlaut_archive(self, filename):
fp = self._open_file(os.path.join(ROOT, 'data', filename), 'rb')
archive = Archive7z(fp)
self.assertEqual(sorted(archive.getnames()), [unicode_string('t\xe4st.txt')])
self.assertEqual(archive.getmember('test.txt'), None)
cf = archive.getmember(unicode_string('t\xe4st.txt'))
self.assertEqual(cf.read(), bytes('This file contains a german umlaut in the filename.', 'ascii'))
cf.reset()
self.assertEqual(cf.read(), bytes('This file contains a german umlaut in the filename.', 'ascii'))
def test_github_33(self):
fp = self._open_file(os.path.join(ROOT, 'data', 'github_33.7z'), 'rb')
archive = Archive7z(fp, password='******')
# Archive only creates an empty file (which gets filtered).
self.assertEqual(archive.getnames(), [])
self.assertEqual(archive.header.files.numfiles, 1)
self.assertEqual([x['filename'] for x in archive.header.files.files], [u'successs.txt'])
cf = archive.getmember('successs.txt')
self.assertEqual(cf, None)
def test_encrypted_wong_password(self):
# test loading of encrypted files with wrong password
fp = self._open_file(os.path.join(ROOT, 'data', 'encrypted.7z'), 'rb')
archive = Archive7z(fp, password='******')
filenames = archive.getnames()
for filename in filenames:
cf = archive.getmember(filename)
self.assertNotEqual(cf, None, filename)
self.assertRaises(WrongPasswordError, cf.checkcrc)
self.assertRaises(WrongPasswordError, cf.read)
def _test_umlaut_archive(self, filename):
fp = file(os.path.join(ROOT, 'data', filename), 'rb')
archive = Archive7z(fp)
self.failUnlessEqual(sorted(archive.getnames()), [u't\xe4st.txt'])
self.failUnlessEqual(archive.getmember(u'test.txt'), None)
cf = archive.getmember(u't\xe4st.txt')
self.failUnlessEqual(
cf.read(), 'This file contains a german umlaut in the filename.')
cf.reset()
self.failUnlessEqual(
cf.read(), 'This file contains a german umlaut in the filename.')
def getArchiveFilenameList(self):
try:
fp = open(self.path, 'r')
szf = Archive7z(fp)
namelist = list(szf.getnames())
fp.close()
return namelist
except Exception as e:
print >> sys.stderr, u"Unable to get 7zipfile list [{0}]: {1}".format(
e, self.path)
return []
def _access_7z(path):
try:
from py7zlib import Archive7z
fp = open(path, 'rb')
archive = Archive7z(fp)
except ImportError:
warnings.warn(
'Python package pylzma required to handle 7-zip archives... archive is inactive'
)
archive = None
return archive
def test_regress_1(self):
# prevent regression bug #1 reported by mail
fp = open(os.path.join(ROOT, 'data', 'regress_1.7z'), 'rb')
archive = Archive7z(fp)
filenames = list(archive.getnames())
self.failUnlessEqual(len(filenames), 1)
cf = archive.getmember(filenames[0])
self.failIfEqual(cf, None)
self.failUnless(cf.checkcrc())
data = cf.read()
self.failUnlessEqual(len(data), cf.size)
def test_bug(self):
# prevent regression bug #1 reported by mail
fp = self._open_file(os.path.join(ROOT, 'data', 'donnees.7z'), 'rb')
archive = Archive7z(fp)
filenames = list(archive.getnames())
self.assertEqual(len(filenames), 1)
cf = archive.getmember(filenames[0])
self.assertNotEqual(cf, None)
self.assertTrue(cf.checkcrc())
data = cf.read()
self.assertEqual(len(data), cf.size)
def getRomInfoListFromFile(self):
if not gotLib7z:
writeError("Sorry, You cannot use 7z file")
return []
listCrc = []
writeInfo("Open 7Z file")
fp = open(self.file, 'rb')
archive = Archive7z(fp)
for name in archive.getnames():
crc = getCRCFromBuffer(archive.getmember(name).read())
listCrc.append(crc)
fp.close()
return listCrc
def test_github_33(self):
fp = self._open_file(os.path.join(ROOT, 'data', 'github_33.7z'), 'rb')
archive = Archive7z(fp, password='******')
# Archive only contains an empty file.
self.assertEqual(archive.getnames(), ['successs.txt'])
self.assertEqual(archive.header.files.numfiles, 1)
self.assertEqual([x['filename'] for x in archive.header.files.files],
[u'successs.txt'])
cf = archive.getmember('successs.txt')
self.assertNotEqual(cf, None)
data = cf.read()
self.assertEqual(len(data), cf.uncompressed)
self.assertEqual(len(data), 0)
self._test_decode_all(archive)
def _test_archive(self, filename):
fp = file(os.path.join(ROOT, 'data', filename), 'rb')
archive = Archive7z(fp)
self.failUnlessEqual(sorted(archive.getnames()),
[u'test/test2.txt', u'test1.txt'])
self.failUnlessEqual(archive.getmember(u'test2.txt'), None)
cf = archive.getmember(u'test1.txt')
self.failUnlessEqual(cf.read(), 'This file is located in the root.')
cf.reset()
self.failUnlessEqual(cf.read(), 'This file is located in the root.')
cf = archive.getmember(u'test/test2.txt')
self.failUnlessEqual(cf.read(), 'This file is located in a folder.')
cf.reset()
self.failUnlessEqual(cf.read(), 'This file is located in a folder.')
def test_github_17(self):
try:
fp = self._open_file(os.path.join(ROOT, 'data', 'ux.stackexchange.com.7z'), 'rb')
except EnvironmentError:
e = sys.exc_info()[1]
if e.errno != errno.ENOENT:
raise
# test is optional
import warnings
warnings.warn("File 'ux.stackexchange.com.7z' was not found, please download manually. Test will be skipped.")
return
archive = Archive7z(fp)
self._test_decode_all(archive)
def readArchiveFile(self, archive_file):
data = ""
fp = open(self.path, 'r')
try:
szf = Archive7z(fp)
data = szf.getmember(archive_file).read()
except Exception as e:
fp.close()
print >> sys.stderr, u"bad 7zip file [{0}]: {1} :: {2}".format(
e, self.path, archive_file)
raise IOError
finally:
fp.close()
return data
def _unzip_7z(fname):
'''This function will decompress a 7zip file, typically
a file ending in .7z (see lidar example in datasets.yml).
The lidar example downloads 7zips and extracts text files
(.gnd) files with this function'''
try:
arc = Archive7z(open(fname, 'rb'))
except:
print('FAILED ON 7Z', fname)
raise
fnames = arc.filenames
files = arc.files
data_dir = os.path.dirname(fname)
for fn, fi in zip(fnames, files):
gnd = path.join(data_dir, os.path.basename(fn))
if not os.path.exists(os.path.dirname(gnd)):
os.mkdir(os.path.dirname(gnd))
with open(gnd, 'w') as f:
f.write(fi.read().decode())
def __init__(self, filename):
"""Opens archive."""
self.file = os.path.realpath(filename)
if not os.path.isfile(self.file) or not os.access(self.file, os.R_OK):
raise IOError("Cannot open %s. No such file." % self.file)
self.filetype = self.get_filetype()
if self.filetype == ZIP:
self.fd = zipfile.ZipFile(self.file, "r")
elif self.filetype == GZIP:
self.fd = gzip.GzipFile(self.file, "rb")
elif self.filetype == BZIP:
self.fd = bz2.BZ2File(self.file, "r")
elif self.filetype == ROM:
self.fd = open(self.file, "rb")
elif self.filetype == RAR:
if HAS_RAR:
rarfile.USE_EXTRACT_HACK = 0
self.fd = rarfile.RarFile(self.file)
elif RAR_CMD:
self.fd = RarCmd(self.file)
else:
raise IOError("rarfile module or rar/unrar is needed for %s." %
self.file)
elif self.filetype == LZMA:
if HAS_7Z:
self.fd = Archive7z(open(self.file, "rb"))
elif LZMA_CMD:
self.fd = LzmaCmd(self.file)
else:
raise IOError("lzma module or 7z is needed for %s." %
self.file)
else:
raise IOError("File %s is not a N64 ROM file." % self.file)
self.namelist = self.get_namelist()
def extract_new_soundpack(self):
self.extracting_new_soundpack = True
if self.downloaded_file.lower().endswith('.7z'):
self.extracting_zipfile = open(self.downloaded_file, 'rb')
self.extracting_archive = Archive7z(self.extracting_zipfile)
self.extracting_infolist = self.extracting_archive.getmembers()
else:
if self.downloaded_file.lower().endswith('.zip'):
archive_class = zipfile.ZipFile
elif self.downloaded_file.lower().endswith('.rar'):
archive_class = rarfile.RarFile
z = archive_class(self.downloaded_file)
self.extracting_zipfile = z
self.extracting_infolist = z.infolist()
self.extract_dir = os.path.join(self.game_dir, 'newsoundpack')
while os.path.exists(self.extract_dir):
self.extract_dir = os.path.join(self.game_dir,
'newsoundpack-{0}'.format('%08x' % random.randrange(16**8)))
os.makedirs(self.extract_dir)
self.extracting_index = 0
main_window = self.get_main_window()
status_bar = main_window.statusBar()
status_bar.busy += 1
extracting_label = QLabel()
status_bar.addWidget(extracting_label, 100)
self.extracting_label = extracting_label
progress_bar = QProgressBar()
status_bar.addWidget(progress_bar)
self.extracting_progress_bar = progress_bar
timer = QTimer(self)
self.extracting_timer = timer
progress_bar.setRange(0, len(self.extracting_infolist))
def timeout():
self.extracting_progress_bar.setValue(self.extracting_index)
if self.extracting_index == len(self.extracting_infolist):
self.extracting_timer.stop()
main_window = self.get_main_window()
status_bar = main_window.statusBar()
status_bar.removeWidget(self.extracting_label)
status_bar.removeWidget(self.extracting_progress_bar)
status_bar.busy -= 1
self.extracting_new_soundpack = False
self.extracting_zipfile.close()
self.extracting_zipfile = None
if self.downloaded_file.lower().endswith('.7z'):
self.extracting_archive = None
if self.install_type == 'direct_download':
download_dir = os.path.dirname(self.downloaded_file)
delete_path(download_dir)
self.move_new_soundpack()
else:
extracting_element = self.extracting_infolist[
self.extracting_index]
self.extracting_label.setText(_('Extracting {0}').format(
extracting_element.filename))
if self.downloaded_file.lower().endswith('.7z'):
destination = os.path.join(self.extract_dir,
*extracting_element.filename.split('/'))
dest_dir = os.path.dirname(destination)
if not os.path.isdir(dest_dir):
os.makedirs(dest_dir)
with open(destination, 'wb') as f:
f.write(extracting_element.read())
else:
self.extracting_zipfile.extract(extracting_element,
self.extract_dir)
self.extracting_index += 1
timer.timeout.connect(timeout)
timer.start(0)
def download_http_finished(self):
self.downloading_file.close()
main_window = self.get_main_window()
status_bar = main_window.statusBar()
status_bar.removeWidget(self.downloading_label)
status_bar.removeWidget(self.dowloading_speed_label)
status_bar.removeWidget(self.downloading_size_label)
status_bar.removeWidget(self.downloading_progress_bar)
status_bar.busy -= 1
if self.download_aborted:
download_dir = os.path.dirname(self.downloaded_file)
delete_path(download_dir)
self.downloading_new_soundpack = False
else:
redirect = self.download_http_reply.attribute(
QNetworkRequest.RedirectionTargetAttribute)
if redirect is not None:
download_dir = os.path.dirname(self.downloaded_file)
delete_path(download_dir)
os.makedirs(download_dir)
self.downloading_file = open(self.downloaded_file, 'wb')
status_bar.busy += 1
redirected_url = urljoin(
self.download_http_reply.request().url().toString(),
redirect.toString())
downloading_label = QLabel()
downloading_label.setText(_('Downloading: {0}').format(
redirected_url))
status_bar.addWidget(downloading_label, 100)
self.downloading_label = downloading_label
dowloading_speed_label = QLabel()
status_bar.addWidget(dowloading_speed_label)
self.dowloading_speed_label = dowloading_speed_label
downloading_size_label = QLabel()
status_bar.addWidget(downloading_size_label)
self.downloading_size_label = downloading_size_label
progress_bar = QProgressBar()
status_bar.addWidget(progress_bar)
self.downloading_progress_bar = progress_bar
progress_bar.setMinimum(0)
self.download_last_read = datetime.utcnow()
self.download_last_bytes_read = 0
self.download_speed_count = 0
progress_bar.setValue(0)
request = QNetworkRequest(QUrl(redirected_url))
request.setRawHeader(b'User-Agent', cons.FAKE_USER_AGENT)
self.download_http_reply = self.qnam.get(request)
self.download_http_reply.finished.connect(
self.download_http_finished)
self.download_http_reply.readyRead.connect(
self.download_http_ready_read)
self.download_http_reply.downloadProgress.connect(
self.download_dl_progress)
else:
# Test downloaded file
status_bar.showMessage(_('Testing downloaded file archive'))
if self.downloaded_file.lower().endswith('.7z'):
try:
with open(self.downloaded_file, 'rb') as f:
archive = Archive7z(f)
except FormatError:
status_bar.clearMessage()
status_bar.showMessage(_('Selected file is a '
'bad archive file'))
self.finish_install_new_soundpack()
return
except NoPasswordGivenError:
status_bar.clearMessage()
status_bar.showMessage(_('Selected file is a '
'password protected archive file'))
self.finish_install_new_soundpack()
return
else:
archive_exception = None
if self.downloaded_file.lower().endswith('.zip'):
archive_class = zipfile.ZipFile
archive_exception = zipfile.BadZipFile
test_method = 'testzip'
elif self.downloaded_file.lower().endswith('.rar'):
archive_class = rarfile.RarFile
archive_exception = rarfile.Error
test_method = 'testrar'
else:
extension = os.path.splitext(self.downloaded_file)[1]
status_bar.clearMessage()
status_bar.showMessage(
_('Unknown downloaded archive format ({extension})'
).format(extension=extension))
self.finish_install_new_soundpack()
return
try:
with archive_class(self.downloaded_file) as z:
test = getattr(z, test_method)
if test() is not None:
status_bar.clearMessage()
status_bar.showMessage(
_('Downloaded archive is invalid'))
self.finish_install_new_soundpack()
return
except archive_exception:
status_bar.clearMessage()
status_bar.showMessage(_('Selected file is a '
'bad archive file'))
self.finish_install_new_soundpack()
return
status_bar.clearMessage()
self.downloading_new_soundpack = False
self.extract_new_soundpack()
