def check(self, fileinfo, logical_component, installation_method):
## Must be a zip file
try:
zf = ZipFile(fileinfo)
zf.testzip()
except BadZipfile, e:
raise forms.ValidationError('File is not an EAR or WAR archive or is corrupted: ' + str(e))
def _run_with_file(self, path: str):
zf = ZipFile(path, "r", ZIP_DEFLATED, allowZip64=False)
zf.testzip()
with open(path, mode="rb") as fp:
self._in_memory.seek(0)
self._in_memory.write(fp.read())
def check(self, fileinfo, logical_component, installation_method):
## Must be a zip file
try:
zf = ZipFile(fileinfo)
zf.testzip()
except BadZipfile, e:
raise forms.ValidationError(e)
def zip_in_memory(self, _path):
"""Compress zip data (bytes) in memory.
:param _path: The path of the directory to be zipped.
"""
try:
# when it is a zip file
if path.isfile(_path):
zf = ZipFile(_path, 'r', ZIP_DEFLATED, False)
zf.testzip()
with open(_path, mode='rb') as fp:
fp.seek(0)
self._in_memory.seek(0)
self._in_memory.write(fp.read())
else:
# root path for figuring out directory of tests
tmp_root = None
with ZipFile(self._in_memory, 'a', ZIP_DEFLATED, False) as zf:
for root, folders, files in walk(_path):
if 'package.json' in files:
tmp_root = root
if tmp_root and root.replace(tmp_root, '') == '/tests':
continue
if root.find('__pycache__') != -1:
continue
if root.find('/.') != -1:
continue
for file in files:
if file.startswith('.'):
continue
full_path = path.join(root, file)
zf.write(full_path)
except ZipException:
raise ZipException
def zip_encrypted(filepath):
try:
zf = ZipFile(filepath)
zf.testzip()
except RuntimeError as e:
if 'encrypted' in str(e):
return True
else:
return False
else:
return False
def parse(self, response):
if (response.status >= 300) and (response.status < 400):
redir = response.css('a::attr(href)').extract_first()
self.dprint("Found redirect: " + redir)
yield scrapy.Request(redir, self.parse, dont_filter=True)
elif response.status == 200:
if response.url[-4:].lower() == '.zip':
fname = response.url.split("/")[-1]
self.dprint("Found zip file: " + fname)
z = ZipFile(BytesIO(response.body))
self.dprint('Testing: ' + fname)
if z.testzip() is not None: self.fatality('Error: ' + fname + ' is corrupted!')
self.dprint("Extracting: " + fname + " -> " + self.odir)
z.extractall(self.odir)
z.close()
self.dprint('Extracted: ' + fname + ' -> ' + self.odir, True)
else:
thisisbase = get_base_url(response)
self.dprint(thisisbase)
relink = response.css('div.download_box a::attr(href)').extract_first()
self.dprint("Found download link: " + relink)
if (sys.version_info > (3, 0)):
dlink = urllib.parse.urljoin(response.url, relink.strip())
else:
dlink = urlparse.urljoin(response.url, relink.strip())
self.dprint("Trying download link: " + dlink)
yield scrapy.Request(dlink, self.parse)
def _getZipFileFromRelease_nowork(self, release):
# download and verify zipfile from the release package
zipball = release.get('zipball_url', None)
if (zipball == None):
raise Exception('Invalid release package: no zipball')
self._debug('Downloading zip file: %s' % zipball)
# zipdata = urlopen(zipball).read()
# https://github.com/six50joe/Indigo/archive/v0.4.6.zip
f = subprocess.Popen(["curl", "-k", zipball], stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=False)
out, err = f.communicate()
self._debug(str(out))
self._debug(str(err))
lines = out.split('\n')
self._debug("Lines: %s" % str(lines))
zipfile = ZipFile(StringIO(out))
self._debug('Verifying zip file (%d bytes)...' % len(zipdata))
if (zipfile.testzip() != None):
raise Exception('Download corrupted')
return zipfile
def update(self, update):
dpath = self.xplanedir + '/Resources/Downloads'
installpath = self.xplanedir + '/Resources/plugins/PythonScripts'
# Broadcast message to all plugins
XPLMSendMessageToPlugin(XPLM_NO_PLUGIN_ID, 0x8000000 | 8090 , long(1))
PI_SendMessageToScript(self, None, 0x8000000 | 8090, 1)
sleep(1)
if not os.path.exists(dpath):
os.mkdir(dpath)
if update['update_type'] == 'direct' and update['update_filename']:
urllib.urlretrieve(update['update_url'], dpath + '/' + update['update_filename'])
copy(dpath + '/' + update['update_filename'], installpath + '/' + update['update_filename'])
print dpath + '/' + update['update_filename'], installpath + '/' + update['update_filename']
elif update['update_type'] == 'zip':
zipfile = dpath + '/._xpjpcUPDATE.zip'
# Download update
urllib.urlretrieve(update['update_url'], zipfile)
zip = ZipFile(zipfile, 'r')
# Check zip file
if not zip.testzip():
# Unzip
unzipdir = dpath + '/' + zip.namelist()[0]
zip.extractall(dpath)
zip.close()
# Move files
self.tcopy(unzipdir, installpath)
rmtree(unzipdir)
os.remove(zipfile)
def force_create(self, archivo, **kw):
"""Sube una entrega en lugar de un alumno"""
instancia = kw['instancia']
entregador = kw['entregador']
archivo = archivo.file.read()
try:
zfile = ZipFile(StringIO(archivo), 'r')
except BadZipfile:
flash(_(u'El archivo ZIP no es válido'))
raise redirect('force_new', kw)
if zfile.testzip() is not None:
flash(_(u'El archivo ZIP tiene errores de CRC'))
raise redirect('force_new',kw)
entregador_id = int(entregador)
instancia = InstanciaDeEntrega.get(int(instancia))
if instancia.ejercicio.grupal:
entregador = Grupo.get(entregador_id)
else:
entregador = AlumnoInscripto.get(entregador_id)
kw['instancia'] = instancia
kw['archivos'] = archivo
kw['entregador'] = entregador
kw['observaciones'] = 'Entrega realizada manualmente por el docente %s' % identity.current.user.shortrepr()
Entrega(**kw)
flash('Se creo una nueva entrega')
raise redirect('list')
def test_tutorials_presentation(self):
section = Section.objects.get(name='Tutorials')
schedule = Schedule.objects.get(section=section)
prop_kind = ProposalKind.objects.get(name__iexact='tutorial')
proposal = PyConTutorialProposalFactory(kind=prop_kind, )
day = Day.objects.create(schedule=schedule, date=date.today())
kind = SlotKind.objects.create(schedule=schedule, label="Foo")
slot = Slot.objects.create(
day=day,
kind=kind,
start=now().time(),
end=now().time(),
)
pres = PresentationFactory(
title=proposal.title,
abstract=proposal.abstract,
section=section,
slot=slot,
cancelled=False,
proposal_base=proposal,
)
rsp = self.client.get(self.url)
self.assertEqual(OK, rsp.status_code)
self.assertEqual('attachment; filename=program_export.zip',
rsp['Content-Disposition'])
zipfile = ZipFile(StringIO(rsp.content), "r")
# Check out the zip - testzip() returns None if no errors found
self.assertIsNone(zipfile.testzip())
fname = "program_export/presentations/csv/tutorials.csv"
file_contents = zipfile.open(fname, "U").read().decode('utf-8')
self.assertIn(pres.title, file_contents)
self.assertIn(pres.abstract, file_contents)
def parse_zipfile_metadata(uploaded_file):
"""
Given a file, extract out the metadata.json, parse, and return it.
"""
try:
zipfile = ZipFile(uploaded_file, 'r')
except (BadZipfile, zlib.error):
raise InvalidExtensionData("Invalid zip file")
if zipfile.testzip() is not None:
raise InvalidExtensionData("Invalid zip file")
total_uncompressed = sum(i.file_size for i in zipfile.infolist())
if total_uncompressed > 5*1024*1024: # 5 MB
raise InvalidExtensionData("Zip file is too large")
try:
metadata = json.load(zipfile.open('metadata.json', 'r'))
except KeyError:
# no metadata.json in archive, raise error
raise InvalidExtensionData("Missing metadata.json")
except ValueError:
# invalid JSON file, raise error
raise InvalidExtensionData("Invalid JSON data")
zipfile.close()
return metadata
def clean_file(self):
def ffile_path(uploaded_file):
''' Converts InMemoryUploadedFile to on-disk file so it will have path. '''
try:
return uploaded_file.temporary_file_path()
except AttributeError:
fileno, path = tempfile.mkstemp()
temp_file = os.fdopen(fileno, 'w+b')
for chunk in uploaded_file.chunks():
temp_file.write(chunk)
temp_file.close()
return path
path = ffile_path(self.cleaned_data['file'])
print path
print path
if path == None:
raise forms.ValidationError('No file selected.')
try: # Comprobacion de que el fichero no esta corrupto
zf = ZipFile(path)
bad_file = zf.testzip()
if bad_file:
raise forms.ValidationError(
_('El fichero "%s" del ZIP esta corrupto.') % bad_file)
zf.close()
except BadZipfile:
raise forms.ValidationError('El fichero subido no es un ZIP.')
return path
def unzipBuild(self, sourceFile, targetFolder):
from zipfile import ZipFile
import os, shutil
sourceFile = str(sourceFile).replace("\\", "/")
zf = ZipFile(file=sourceFile)
self.logger.debug("Go to test source zip file")
zf.testzip()
# Empty target Folder
try:
shutil.rmtree(targetFolder)
self.logger.debug("Remove folder " + targetFolder)
except Exception, e:
self.logger.debug(e)
def _get_carparks_xml_from_zip(url=CARPARKS_ZIP_URL, index_xml=INDEX_XML_FILE_NAME):
res = request.urlopen(url=url)
# Validate a successful HTTP call with status 200.
if not res.status == 200:
raise Exception('Call to \'{0!s}\' failed with status code {1!s}.'.format(url, res.status))
# Convert the downloaded byte stream to a file-like in-memory object.
zip_file = BytesIO(res.read())
# Validate the file-like object contains a valid zip file.
if not is_zipfile(zip_file):
raise Exception('The URL \'{0!s}\' did not return a valid zip file.'.format(url))
# Convert to an actual ZipFile object.
zip = ZipFile(zip_file, 'r')
# Fail if the returned zip file is corrupt.
if zip.testzip():
raise Exception('Zip file from \'{0!s}\' was corrupt.'.format(url))
# Create a list of filenames to process.
xml_filenames = _get_filenames_from_index_xml(index=zip.read(index_xml))
# Validate there is at least 1 file to process.
if len(xml_filenames) < 1:
raise Exception('No XML files listed in {0!s}!'.format(index_xml))
# Create and populate a dictionary with filenames and contents.
return_str_xmls = dict()
for filename in xml_filenames:
return_str_xmls[filename] = zip.read(filename)
# Return the dictionary.
return return_str_xmls
def __init__(self, path):
zipfile = ZipFile(path, mode='r')
if zipfile.testzip() is not None:
raise BadQMODFileException("'" + path + "' failed zip test")
files = zipfile.namelist()
if 'qmod.ini' not in files:
raise BadQMODFileException("Missing 'qmod.ini'")
files.remove('qmod.ini')
dirs = set([Path(x).parts[0] for x in files])
if len(dirs) > 1:
raise BadQMODFileException('Improper directory structure')
ini_string = zipfile.read('qmod.ini').decode('utf-8')
config = ConfigParser()
config.read_string(ini_string)
self.name = config['general']['name']
self.shortdesc = config['general']['shortdesc']
self.version = config['general']['version']
self.longdesc = ' '.join(
[line for line in config['longdesc'].values()])
self.gamedir = config['general']['gamedir']
self.datafiles = files
self.zipfile = zipfile
def update(self, update):
dpath = self.xplanedir + '/Resources/Downloads'
installpath = self.xplanedir + '/Resources/plugins/PythonScripts'
print update
if not os.path.exists(dpath):
os.mkdir(dpath)
if update['update_type'] == 'direct' and update['update_filename']:
urllib.urlretrieve(update['update_url'],
dpath + '/' + update['update_filename'])
copy(dpath + '/' + update['update_filename'],
installpath + '/' + update['update_filename'])
print dpath + '/' + update[
'update_filename'], installpath + '/' + update[
'update_filename']
elif update['update_type'] == 'zip':
zipfile = dpath + '/._xpjpcUPDATE.zip'
# Download update
urllib.urlretrieve(update['update_url'], zipfile)
zip = ZipFile(zipfile, 'r')
# Check zip file
if not zip.testzip():
# Unzip
unzipdir = dpath + '/' + zip.namelist()[0]
zip.extractall(dpath)
zip.close()
# Move files
self.tcopy(unzipdir, installpath)
rmtree(unzipdir)
os.remove(zipfile)
def check(path):
"""Used to check a snapshot before installing.
Assure all files extract to any subdirectories of a sakura system
directory, e.g., cgi/, functions/, content/.
Args:
path (str): The path to the snapshot.
Notes:
Maybe this should take a zipfile object
Returns:
None
"""
zip_file = ZipFile(path, 'r')
print zip_file.comment
crc_check = zip_file.testzip()
zip_file.close()
if crc_check:
raise Exception(crc_check)
else:
print 'passed CRC and file check'
sanity_check(path)
return None
def _modify_zip_file(self, zip_path, filename, replacement_content=None):
"""
Replace the contents of a file inside a ZIP-container. The original file
will not be overwritten. A BytesIO() file will be returned containing
the new ZIP-container with the file replaced.
:param zip_path The path to the ZIP-File to be modified
:param filename Filename inside the container to be modified
:param replacement_content The new container which will be stored in `filename`
:return BytesIO contain the modified zip file.
"""
vuln_zipfile = BytesIO()
zw = ZipFile(vuln_zipfile, 'w')
zr = ZipFile(zip_path)
names_to_copy = set(zr.namelist()) - {filename, }
for name in names_to_copy:
content = zr.read(name)
zw.writestr(name, content)
if replacement_content:
zw.writestr(filename, replacement_content)
zw.close()
vuln_zipfile.seek(0)
x = ZipFile(vuln_zipfile, 'r')
self.assertEqual(x.testzip(), None)
vuln_zipfile.seek(0)
return vuln_zipfile
def test_tutorials_presentation(self):
section = Section.objects.get(name='Tutorials')
schedule = Schedule.objects.get(section=section)
prop_kind = ProposalKind.objects.get(slug='tutorial')
proposal = PyConTutorialProposalFactory(
kind=prop_kind,
)
day = Day.objects.create(schedule=schedule, date=date.today())
kind = SlotKind.objects.create(schedule=schedule, label="Foo")
slot = Slot.objects.create(
day=day,
kind=kind,
start=now().time(),
end=now().time(),
)
pres = PresentationFactory(
title=proposal.title,
abstract=proposal.abstract,
section=section,
slot=slot,
cancelled=False,
proposal_base=proposal,
)
rsp = self.client.get(self.url)
self.assertEqual(OK, rsp.status_code)
self.assertEqual('attachment; filename=program_export.zip', rsp['Content-Disposition'])
zipfile = ZipFile(StringIO(rsp.content), "r")
# Check out the zip - testzip() returns None if no errors found
self.assertIsNone(zipfile.testzip())
fname = "program_export/presentations/csv/tutorials.csv"
file_contents = zipfile.open(fname, "U").read().decode('utf-8')
self.assertIn(pres.title, file_contents)
self.assertIn(pres.abstract, file_contents)
def validate_zip_integrity(uploaded_file):
"""
If uploaded file is a .zip archive, check its integrity and the
integrity of the files it contains.
In case of a corrupted archive the `ZipFile` constructor raises
IOError. To check the integrity of the files contained in the
archive, the `ZipFile.testzip()` function is used.
If the uploaded file appears to be a .zip archive (because its
extension is `.zip`), but actually isn't, the `ZipFile`
constructor raises `BadZipFile`. Because this case is covered by
the MIME type validator, the function does not raise a
ValidationError in this case.
"""
if uploaded_file.name.endswith("zip"):
corrupted_file = None
try:
archive = ZipFile(get_tmp_path(uploaded_file.name))
corrupted_file = archive.testzip()
except IOError:
raise ValidationError(UploadFormErrors.ZIP_INTEGRITY)
except BadZipFile:
pass
if corrupted_file:
raise ValidationError(UploadFormErrors.FILES_INTEGRITY)
def test_returns_zip_file(self):
fake_dt = timezone.make_aware(datetime.datetime(2018, 1, 12, 11, 00))
with patch.object(timezone, 'now', return_value=fake_dt):
# Save Pixel Sets in user session
pixel_sets = factories.PixelSetFactory.create_batch(2)
data = {'pixel_sets': [str(p.id) for p in pixel_sets]}
self.client.post(reverse('explorer:pixelset_select'),
data,
follow=True)
response = self.client.get(self.url)
self.assertEqual(response.status_code, 200)
self.assertEqual(response['Content-Type'], 'application/zip')
self.assertEqual(
response['Content-Disposition'],
'attachment; filename={}'.format(
PixelSetExportView.get_export_archive_filename()))
try:
zip = ZipFile(BytesIO(response.content), 'r')
self.assertIsNone(zip.testzip())
finally:
zip.close()
def parse_zipfile_metadata(uploaded_file):
"""
Given a file, extract out the metadata.json, parse, and return it.
"""
try:
zipfile = ZipFile(uploaded_file, 'r')
except (BadZipfile, zlib.error):
raise InvalidExtensionData("Invalid zip file")
if zipfile.testzip() is not None:
raise InvalidExtensionData("Invalid zip file")
total_uncompressed = sum(i.file_size for i in zipfile.infolist())
if total_uncompressed > 5 * 1024 * 1024: # 5 MB
raise InvalidExtensionData("Zip file is too large")
try:
metadata = json.load(zipfile.open('metadata.json', 'r'))
except KeyError:
# no metadata.json in archive, raise error
raise InvalidExtensionData("Missing metadata.json")
except ValueError:
# invalid JSON file, raise error
raise InvalidExtensionData("Invalid JSON data")
zipfile.close()
return metadata
def download_category(category: str) -> list[str]:
"""
Download and extract the HTML data tables of a specific term category.
Returns a list of paths of the extracted files.
"""
# Download the ZIP archive from NAER website
archive_url = NAER_ARCHIVE_URL.format(category=category)
with requests.get(archive_url) as r:
# Load the ZIP file and check its header
zip_archive = ZipFile(BytesIO(r.content))
assert zip_archive.testzip() is None
# Recursively download and extract the HTML
file_paths = []
for zip_item in zip_archive.infolist():
# Fix the file extension
filename = zip_item.filename
ext_index = filename.index('.xls') # It’s actually HTML. LIES!
# Change the extension and write them all into target directory
file_path = os.path.join('data', 'html', filename[:ext_index] + '.html')
with open(file_path, 'wb') as f:
f.write(zip_archive.read(zip_item))
file_paths.append(file_paths)
print(file_path)
return file_paths
def openFile(self, file_descriptor):
"""
Load all files in the zipped OpenOffice document
"""
# Try to unzip the Open Office doc
try:
oo_unzipped = ZipFile(file_descriptor, mode="r")
except Exception as e:
LOG('ERP5OOo', DEBUG, 'Error in openFile', error=True)
raise CorruptedOOoFile(e)
# Test the integrity of the file
if oo_unzipped.testzip() is not None:
raise CorruptedOOoFile('Invalid zip file')
# Get the filename
self.filename = getattr(file_descriptor, 'filename',
'default_filename')
# List and load the content of the zip file
for name in oo_unzipped.namelist():
self.oo_files[name] = oo_unzipped.read(name)
oo_unzipped.close()
# Get the main content and style definitions
self.oo_content_dom = etree.XML(self.oo_files["content.xml"])
self.oo_styles_dom = etree.XML(self.oo_files["styles.xml"])
def checkZip(file, purge=False, verbose=False):
"""
Use Zip to see if the file is valid and prints the name of failed files to
stdout. If purge is set to true it will also delete the file.
In verbose mode it will print a dot for every good file as well.
"""
global zip
try:
zip = ZipFile(file)
if zip.testzip() == None:
if verbose:
sys.stdout.write(".")
sys.stdout.flush()
else:
if verbose:
print()
print("File", file, "failed with corrupted files")
if purge:
os.remove(file)
except BadZipfile:
if verbose:
print()
print("File %s failed with flying colors. Not a valid zip." % file)
if purge:
os.remove(file)
finally:
zip.close()
def test_archive(self):
for (example, ExampleDataset) in examples.examples:
filename = create_archive(example.data_csv, example.sli_manifest)
zip_file = ZipFile(filename, "r")
self.assertEquals(None, zip_file.testzip())
self.assertEquals(zip_file.namelist(), ['data.csv', 'upload_info.json'])
zip_file.close()
os.remove(filename)
def test_zipped_excluded_directory(self):
zippedRoot = self.get_zipped_root('1/')
expected_entries = ['2/2-1/2-1.txt']
zf = ZipFile(zippedRoot)
self.assertTrue(zf.testzip() is None)
for elem in expected_entries:
self.assertTrue(elem in zf.namelist())
self.assertEqual(len(expected_entries), len(zf.namelist()))
zf.close()
def test_zip_path():
runner = CliRunner()
with runner.isolated_filesystem():
result = runner.invoke(cli, ['pull', '-f', 519629, '-p', 'test.zip'])
assert result.exit_code == 0
assert exists('test.zip')
zip_file = ZipFile('test.zip')
assert zip_file.testzip() is None
def decrypt_data(encrypted_string, key, string_file_mode=False, key_file_mode=False):
"""Method that takes either the key or the encrypted string as a
string or can the key and encrypted string a as file and decrypts
the string using the provided string. NOTE** In order to use the the key.dat file
you must first also be able to unzip it using a password."""
print("Starting Decryption...")
if key_file_mode:
if ".zip" in key:
zf = ZipFile(key)
try:
if zf.testzip() == None:
ZipFile(key).extractall()
print("Successfully extracted, please use the key file \
with the .dat extension file as your key and try again.\n")
exit(0)
except:
print("Key.zip is encrypted!\n")
_unzip_file(key, input("Please enter the password to unzip the key file and try again.\n"))
else:
my_key = key
with open(my_key, 'r') as key_data:
my_key = key_data.read()
else:
my_key = key
if string_file_mode:
my_string = encrypted_string
with open(my_string, 'r') as string_data:
my_string = string_data.read()
else:
my_string = encrypted_string
my_string_num_list = my_string
my_key_num_list = _string_converter(my_key)[2:]
print("Decrypting file...please wait, this may take a while depending on file size.")
decrypt_list = []
for j in range(2, len(my_string_num_list)):
index = j % len(my_key_num_list)
decrypt_list.append(int(my_string_num_list[j]) ^ int(my_key_num_list[index]))
decrypted_string = int("0b" + "".join((str(i) for i in decrypt_list)), 2)
if version_info >= (3, 0):
message = decrypted_string.to_bytes((decrypted_string.bit_length() + 7) // 8, 'big').decode()
else:
message = unhexlify('%x' % decrypted_string)
with open("decrypted_message.txt", 'w') as out_message:
out_message.write(message)
print("Decryption Complete.")
return message
def test_003_user_can_download_credentials(self):
buf = self.get_signed_zip(test_username)
output = open(ZIP_FILENAME, 'w')
output.write(buf)
output.close()
zip = ZipFile(ZIP_FILENAME, 'a', ZIP_DEFLATED)
bad = zip.testzip()
zip.close()
self.failIf(bad)
def test_upload(self):
example = examples.examples[0][0]
connection = Connection(username, password, debug=0)
dir_name = connection.webdav.upload(example.data_csv, example.sli_manifest)
self.assert_(len(dir_name) > 0)
self.assert_(connection.webdav.request('/uploads/%s' % dir_name))
uploaded_file = connection.webdav.request('/uploads/%s/upload.zip' % dir_name)
tmp_file = write_tmp_file(uploaded_file.read())
zip_file = ZipFile(tmp_file, "r")
self.assertEquals(None, zip_file.testzip())
self.assertEquals(zip_file.namelist(), ['data.csv', 'upload_info.json'])
zip_file.close()
os.remove(tmp_file)
def is_valid_compressed(file):
"""Check tar gz or zip is valid."""
try:
archive = ZipFile(file, 'r')
try:
corrupt = archive.testzip()
except zlib_error:
corrupt = True
archive.close()
except BadZipfile:
corrupt = True
return not corrupt
def check_archive(self, test_name, archive_path, keep_comments=False):
got = ZipFile(archive_path, 'r')
want = ZipFile('{name}-want.{ext}'.format(name=test_name,
ext=archive_path.split('.')[-1]))
try:
if got.testzip() is not None:
return ("Not valid zip file", "")
got_mf = self.read_archive(got, keep_comments)
want_mf = self.read_archive(want, keep_comments)
finally:
got.close()
want.close()
return (got_mf, want_mf)
def check_archive(self, test_name, archive_path, keep_comments=False):
got = ZipFile(archive_path, 'r')
want = ZipFile('{name}-want.{ext}'.format(
name=test_name, ext=archive_path.split('.')[-1]))
try:
if got.testzip() is not None:
return ("Not valid zip file", "")
got_mf = self.read_archive(got, keep_comments)
want_mf = self.read_archive(want, keep_comments)
finally:
got.close()
want.close()
return (got_mf, want_mf)
def __init__(self, path):
try:
archive = ZipFile(path, mode='r')
except BadZipFile:
name = Path(path).name
raise BadQMODFileError(f"'{name}' is not in ZIP compressed format")
if archive.testzip() is not None:
raise BadQMODFileError(f"'{path}' failed zip test")
# Some QMODs wrap everything in a top-level directory, possibly due to
# the behaviour of OS ZIPing tools. This is not how QMODs are specced,
# but the following code can handle it.
wrapper = commonprefix(archive.namelist())
files = archive.namelist()
qmod_ini_name = wrapper + 'qmod.ini'
if qmod_ini_name not in files:
raise BadQMODFileError("Missing 'qmod.ini'")
files.remove(qmod_ini_name)
# FIXME check it's a dir
dirs = set([Path(x).parts[0] for x in files])
if len(dirs) > 1:
raise BadQMODFileError('Improper directory structure')
try:
ini_string = archive.read(qmod_ini_name).decode('utf-8')
except Exception as err:
raise BadQMODFileError(f"Couldn't read/decode 'qmod.ini': {err}")
try:
config = ConfigParser()
config.read_string(ini_string)
except Exception as err:
raise BadQMODFileError(f"'qmod.ini' is invalid: {err}")
try:
self.name = config['general']['name']
self.shortdesc = config['general']['shortdesc']
self.version = config['general']['version']
self.longdesc = ' '.join(
[line for line in config['longdesc'].values()])
self.gamedir = config['general']['gamedir']
# FIXME check gamedir matches above dir
except KeyError as err:
raise BadQMODFileError(f"'qmod.ini' is missing section/key '{err}'")
self.archive = archive
self.wrapper = wrapper
def openZip(file, itest=None, checkzip=True, pwd=''):
zfile = ZipFile(str(file), mode='r')
contents = ZipFileContents()
names = zfile.namelist()
if itest and len(names) != len(itest) and set(names) != set(itest):
raise ZipFileIntegrityError(file, itest, names,
len(names) == len(itest),
set(names) == set(itest))
elif checkzip:
dtest = zfile.testzip()
if dtest is not None: raise ZipFileDamagedError(file, dtest)
for name in zfile.namelist():
contents[name] = zfile.open(name, pwd=bytes(pwd, 'utf-8'))
return contents
def test_generates_valid_zip_file(self):
zipdata = BytesIO()
zip_generator = plugin_zip_generator(build_plugin_config())
for zipdata_part in zip_generator:
zipdata.write(zipdata_part)
self.assertGreater(len(zipdata.getvalue()), 0)
parsed_zip = ZipFile(zipdata, 'r')
self.assertIsNone(parsed_zip.testzip())
self.assertGreater(len(parsed_zip.infolist()), 0)
for info in parsed_zip.infolist():
octal_perms = '%o' % (info.external_attr >> 16)
self.assertIn(octal_perms, ['660', '770'])
def test_proc(dir_path, queue):
while True:
zip_path = queue.get()
if zip_path is None:
queue.put(None)
break
rel_path = zip_path.relative_to(dir_path)
zip_file = ZipFile(zip_path, 'r', allowZip64=True)
result = zip_file.testzip()
if result is None:
print(f'[+] {rel_path}')
else:
print(f'[-] {rel_path}, {result}')
def clean_zip_file(self):
''' Checks if zip file is not corrupted, stores in-memory uploaded file
to disk and returns path to stored file.
'''
path = _file_path(self.cleaned_data['zip_file'])
try:
zf = ZipFile(path)
bad_file = zf.testzip()
if bad_file:
raise forms.ValidationError(_('"%s" in the .zip archive is corrupt.') % bad_file)
zf.close()
except BadZipfile:
raise forms.ValidationError(_('Uploaded file is not a zip file.'))
return path
def test_filters_omics_units(self):
pixel_set = factories.PixelSetFactory.create()
pixels = factories.PixelFactory.create_batch(2, pixel_set=pixel_set)
# select pixel set, otherwise we cannot set omics units
self.client.post(reverse('explorer:pixelset_select'),
{'pixel_sets': [pixel_set.id]},
follow=True)
response = self.client.get(self.url)
self.assertIsNone(
self.get_search_terms(self.client.session, default=None))
selected_pixel = pixels[1]
# set search terms in session
response = self.client.post(
reverse('explorer:pixelset_selection'), {
'search_terms': selected_pixel.omics_unit.reference.identifier,
},
follow=True)
fake_dt = timezone.make_aware(datetime.datetime(2018, 1, 12, 11, 00))
with patch.object(timezone, 'now', return_value=fake_dt):
response = self.client.get('{}?{}=1'.format(
self.url,
PixelSetExportView.SUBSET_QUERY_PARAM,
))
self.assertEqual(response.status_code, 200)
self.assertEqual(response['Content-Type'], 'application/zip')
self.assertEqual(
response['Content-Disposition'],
'attachment; filename={}'.format(
PixelSetExportView.get_export_archive_filename()))
try:
zip = ZipFile(BytesIO(response.content), 'r')
self.assertIsNone(zip.testzip())
with zip.open(PIXELSET_EXPORT_PIXELS_FILENAME) as pixels_file:
pixels_csv = pandas.read_csv(pixels_file)
self.assertEqual(len(pixels_csv.index), 1)
finally:
zip.close()
def update(self, update):
dpath = self.xplanedir + '/Resources/Downloads'
installpath = self.xplanedir + '/Resources/plugins/PythonScripts'
if not os.path.exists(dpath):
os.mkdir(dpath)
if update['update_type'] == 'direct' and update['update_filename']:
if not self.download(update['update_url'],
dpath + '/' + update['update_filename']):
return False
self.printStatus('Installing files:')
copy(dpath + '/' + update['update_filename'],
installpath + '/' + update['update_filename'])
self.printStatus('.', False)
return True
elif update['update_type'] == 'zip':
zipfile = dpath + '/._xpjpcUPDATE.zip'
# Download update
self.printStatus('Downloading: \n%s' % update['update_url'])
try:
urllib.urlretrieve(update['update_url'], zipfile)
except:
self.printStatus('ERROR: ')
return False
self.printStatus('.', False)
zip = ZipFile(zipfile, 'r')
self.printStatus('Unpacking: %s' % update['name'])
# Check zip file
if not zip.testzip():
# Unzip
unzipdir = dpath + '/' + zip.namelist()[0]
zip.extractall(dpath)
zip.close()
# Move files
self.tcopy(unzipdir, installpath)
rmtree(unzipdir)
os.remove(zipfile)
self.printStatus('.', False)
return True
def _getZipFileFromRelease(self, release):
# download and verify zipfile from the release package
zipball = release.get('zipball_url', None)
if (zipball == None):
raise Exception('Invalid release package: no zipball')
self._debug('Downloading zip file: %s' % zipball)
zipdata = urlopen(zipball).read()
zipfile = ZipFile(StringIO(zipdata))
self._debug('Verifying zip file (%d bytes)...' % len(zipdata))
if (zipfile.testzip() != None):
raise Exception('Download corrupted')
return zipfile
def clean_archive(self):
value = self.cleaned_data['archive']
if not is_zipfile(value):
raise ValidationError(_('ZIP file must be uploaded.'))
elif not value.name.endswith('.zip'):
raise ValidationError(_("File must have '.zip' extension."))
zip_file = ZipFile(value)
invalid_file = zip_file.testzip()
if invalid_file is not None:
raise ValidationError(
_('File {filename} in archive {archive_name} has an invalid type.'
.format(filename=invalid_file,
archive_name=zip_file.filename)))
return value
def _getZipFileFromRelease(self, release):
# download and verify zipfile from the release package
zipball = release.get("zipball_url", None)
if zipball == None:
raise Exception("Invalid release package: no zipball")
self._debug("Downloading zip file: %s" % zipball)
zipdata = urlopen(zipball).read()
zipfile = ZipFile(StringIO(zipdata))
self._debug("Verifying zip file (%d bytes)..." % len(zipdata))
if zipfile.testzip() != None:
raise Exception("Download corrupted")
return zipfile
def test_get_ntfs_file_ok(app, coverage_obj, fixture_dir):
with get_valid_ntfs_memory_archive() as (ntfs_file_name, ntfs_zip_memory):
files = {"file": (ntfs_zip_memory, ntfs_file_name)}
with requests_mock.Mocker() as m:
m.post("http://tyr.prod/v0/instances/test", text="ok")
raw = app.post("/coverages/test/environments/production/data_update", data=files)
r = to_json(raw)
assert r["message"].startswith("Valid fusio file provided")
raw = app.get("/coverages/test/environments/production/data/ntfs")
assert raw.status_code == 200
assert raw.mimetype == "application/zip"
data = raw.get_data()
with BytesIO(data) as ntfs_zip_memory:
ntfs_zip = ZipFile(ntfs_zip_memory, "r", ZIP_DEFLATED, False)
assert ntfs_zip.testzip() is None
ntfs_zip.close()
def _getZipFileFromRelease(self, release):
# download and verify zipfile from the release package
zipball = release.get('zipball_url', None)
if (zipball == None):
raise Exception('Invalid release package: no zipball')
self.logger.debug('Downloading zip file: %s' % zipball)
zipdata = urlopen(zipball).read()
zipfile = ZipFile(StringIO(zipdata))
self.logger.debug('Verifying zip file (%d bytes)...' % len(zipdata))
if (zipfile.testzip() != None):
raise Exception('Download corrupted')
return zipfile
def update(self, update):
dpath = self.xplanedir + '/Resources/Downloads'
installpath = self.xplanedir + '/Resources/plugins/PythonScripts'
if not os.path.exists(dpath):
os.mkdir(dpath)
if update['update_type'] == 'direct' and update['update_filename']:
if not self.download(update['update_url'], dpath + '/' + update['update_filename']):
return False
self.printStatus('Installing files:')
copy(dpath + '/' + update['update_filename'], installpath + '/' + update['update_filename'])
self.printStatus('.', False)
return True
elif update['update_type'] == 'zip':
zipfile = dpath + '/._xpjpcUPDATE.zip'
# Download update
self.printStatus('Downloading: \n%s' % update['update_url'])
try:
urllib.urlretrieve(update['update_url'], zipfile)
except:
self.printStatus('ERROR: ')
return False
self.printStatus('.', False)
zip = ZipFile(zipfile, 'r')
self.printStatus('Unpacking: %s' % update['name'])
# Check zip file
if not zip.testzip():
# Unzip
unzipdir = dpath + '/' + zip.namelist()[0]
zip.extractall(dpath)
zip.close()
# Move files
self.tcopy(unzipdir, installpath)
rmtree(unzipdir)
os.remove(zipfile)
self.printStatus('.', False)
return True
def test_scheduled_events_export(self):
self.unpublished_special = ScheduledEventFactory(published=False)
self.published_special = ScheduledEventFactory(published=True)
rsp = self.client.get(self.url)
self.assertEqual(OK, rsp.status_code)
self.assertEqual('attachment; filename=program_export.zip', rsp['Content-Disposition'])
zipfile = ZipFile(StringIO(rsp.content), "r")
# Check out the zip - testzip() returns None if no errors found
self.assertIsNone(zipfile.testzip())
fname = "program_export/scheduled_events/csv/scheduled_events_schedule.csv"
file_contents = zipfile.open(fname, "U").read().decode('utf-8')
self.assertIn(self.published_special.name, file_contents)
self.assertIn(self.published_special.description, file_contents)
self.assertNotIn(self.unpublished_special.name, file_contents)
self.assertNotIn(self.unpublished_special.description, file_contents)
def test_make_archive(self):
# make archive from sample directory
directory = CharmDirectory(sample_directory)
f = tempfile.NamedTemporaryFile(suffix=".charm")
directory.make_archive(f.name)
# open archive in .zip-format and assert integrity
from zipfile import ZipFile
zf = ZipFile(f.name)
self.assertEqual(zf.testzip(), None)
# assert included
included = [info.filename for info in zf.infolist()]
self.assertEqual(
set(included),
set(("metadata.yaml", "empty/", "src/", "src/hello.c",
"config.yaml", "hooks/", "hooks/install", "revision")))
def install(language, directory = config.default_dic_path,
repos = config.default_repository):
'''
Download and install a dictionary file.
language: a string of the form 'll_CC'. Example: 'en_US' for English, USA
directory: the installation directory. Defaults to the
value given in config.py. After installation this is the package root of 'hyphen'
repos: the url of the dictionary repository. (Default: as declared in config.py;
after installation this is the OpenOffice repository for dictionaries.).'''
url = ''.join((repos, 'hyph_', language, '.zip'))
s = urllib2.urlopen(url).read()
z = ZipFile(StringIO(s))
if z.testzip():
raise IOError('The ZIP archive containing the dictionary is corrupt.')
dic_filename = ''.join(('hyph_', language, '.dic'))
dic_str = z.read(dic_filename)
dest = open('/'.join((directory, dic_filename)), 'w')
dest.write(dic_str)
dest.close()
def installPlugin(self, url):
url = 'http://www.xpluginsdk.org/downloads/latest/Python27/PythonInterface.zip'
dpath = self.xplanedir + '/Resources/Downloads'
installpath = self.xplanedir + '/Resources/plugins'
zipfile = dpath + '/PythonInterface.zip'
# Download update
if not self.download(url, zipfile):
return False
zip = ZipFile(zipfile, 'r')
# Check zip file
if not zip.testzip():
# Unzip
zip.extractall(installpath)
zip.close()
def archive(self, mask='*', dryrun=1):
for fn in self.targets(mask):
if not islink(fn):
arc = self.arc(fn)
if exists(arc):
print("already exists: {}".format(badf))
continue
if is_zipfile(fn):
print("checking: {}".format(fn))
zf = ZipFile(fn)
badf = zf.testzip()
print("badf = {}".format(badf))
if badf is None:
print("Moving: {} -> {}".format(fn, arc))
if not dryrun:
shutil.move(fn, arc)
os.chmod(arc, stat.S_IRUSR|stat.S_IRGRP|stat.S_IROTH)
print("Link: {} -> {}".format(fn, arc))
if not dryrun: os.symlink(arc, fn)
def validate_response(self, rsp, names_and_sizes):
# Ensure a response from the view looks right, contains a valid
# zip archive, has files with the right names and sizes.
self.assertEqual("application/zip", rsp['Content-type'])
prefix = settings.CONFERENCE_URL_PREFIXES[settings.CONFERENCE_ID]
self.assertEqual(
'attachment; filename="pycon_%s_sponsorlogos.zip"' % prefix,
rsp['Content-Disposition'])
zipfile = ZipFile(StringIO(rsp.content), "r")
# Check out the zip - testzip() returns None if no errors found
self.assertIsNone(zipfile.testzip())
# Compare contents to what is expected
infolist = zipfile.infolist()
self.assertEqual(len(names_and_sizes), len(infolist))
for info, name_and_size in zip(infolist, names_and_sizes):
name, size = name_and_size
self.assertEqual(name, info.filename)
self.assertEqual(size, info.file_size)
def unzip(file_obj):
"""
Take a path to a zipfile and checks if it is a valid zip file
and returns...
"""
files = []
# TODO: implement try-except here
zip = ZipFile(file_obj)
bad_file = zip.testzip()
if bad_file:
raise Exception('"%s" in the .zip archive is corrupt.' % bad_file)
infolist = zip.infolist()
for zipinfo in infolist:
if zipinfo.filename.startswith('__'): # do not process meta files
continue
file_obj = SimpleUploadedFile(name=zipinfo.filename, content=zip.read(zipinfo))
files.append((file_obj, zipinfo.filename))
zip.close()
return files
class CZIP:
def __init__(self, zipfilepath, fmode=FWrite):
self._zipfile = ZipFile(zipfilepath, fmode)
def __del__(self):
self.close()
def zipdir(self, path, progress, ttl=Empty, template=Empty):
flcount = 0
arch = [itm for itm in DOS.walk(path)]
substep = 100.0 / len(arch)
for root, dirs, files in arch:
lenf = len(files)
if lenf : stepv = substep / lenf
else : progress.step(ttl, substep)
for file in files:
progress.step(ttl if ttl else file, stepv)
if template and not self.isTempl(file, template) : continue
fullpath = DOS.join(root, file)
base = fullpath.replace(path, Empty)
self._zipfile.write(esys(de(fullpath)), esys(de(base)))
flcount += 1
del progress
return flcount
def isTempl(self, file, template):
for tmpl in template:
if file.find(tmpl) != -1 : return True
return False
def unzip(self, path):
self._zipfile.extractall(path)
def crc(self):
try : rc = True if self._zipfile.testzip() is None else False
except : rc = False
return rc
def close(self):
self._zipfile.close()
