def _yamlfiles(zipfile):
"""Helper to get a yaml file (according to POLICYD_VALID_EXTS extensions)
and the infolist item from a zipfile.
:param zipfile: the zipfile to read zipinfo items from
:type zipfile: zipfile.ZipFile
:returns: generator of (name, ext, filename, info item) for each self-identified
yaml file.
:rtype: List[(str, str, str, zipfile.ZipInfo)]
"""
l = []
for infolist_item in zipfile.infolist():
try:
if infolist_item.is_dir():
continue
except AttributeError:
# fallback to "old" way to determine dir entry for pre-py36
if infolist_item.filename.endswith('/'):
continue
_, name_ext = os.path.split(infolist_item.filename)
name, ext = os.path.splitext(name_ext)
ext = ext.lower()
if ext and ext in POLICYD_VALID_EXTS:
l.append((name, ext, name_ext, infolist_item))
return l
def read_metadata(zipfile):
for info in zipfile.infolist():
basename = os.path.basename(info.filename)
if basename == METADATA_FILENAME:
meta_file = zipfile.read(info.filename)
return kbjson.loads(meta_file)
return {}
def zipfile_to_infos(self, zipfile: zipfile.ZipFile) -> ArchiveInfo:
archive_info = ArchiveInfo()
for ziplineinfo in zipfile.infolist():
fileinfo = FileInfo()
fileinfo.last_modification = datetime(
year=ziplineinfo.date_time[0],
month=ziplineinfo.date_time[1],
day=ziplineinfo.date_time[2],
hour=ziplineinfo.date_time[3],
minute=ziplineinfo.date_time[4],
second=ziplineinfo.date_time[5]
)
fileinfo.name = ziplineinfo.filename
fileinfo.size = ziplineinfo.file_size
fileinfo.size__compressed = ziplineinfo.compress_size
# if info.is_dir():
# fileinfo.type = FileInfo.DIR
# else:
# fileinfo.type = FileInfo.FILE
archive_info.files.append(fileinfo)
archive_info.size += fileinfo.size
archive_info.size__compressed += fileinfo.size__compressed
if not archive_info.last_modification \
or archive_info.last_modification < fileinfo.last_modification:
archive_info.last_modification = fileinfo.last_modification
return archive_info
def read_metadata(zipfile):
for info in zipfile.infolist():
basename = os.path.basename(info.filename)
if basename == METADATA_FILENAME:
meta_file = zipfile.read(info.filename)
return kbjson.loads(meta_file)
return {}
def read_metadata(zipfile):
"""Reads and returns metadata from a backup zipfile."""
for info in zipfile.infolist():
basename = os.path.basename(info.filename)
if basename == METADATA_FILENAME:
meta_file = zipfile.read(info.filename)
return kbjson.loads(meta_file)
return {}
def read_metadata(zipfile):
"""Reads and returns metadata from a backup zipfile."""
for info in zipfile.infolist():
basename = os.path.basename(info.filename)
if basename == METADATA_FILENAME:
meta_file = zipfile.read(info.filename)
return kbjson.loads(meta_file)
return {}
def _get_labels(zipfile, start_index):
print("Extracting labels\n")
labels = []
for zipinfo in zipfile.infolist()[1:]:
file_name = zipinfo.filename
ind = file_name.find("-", start_index)
labels.append(file_name[start_index:ind])
return labels
def _safe_extract_all(self, zipfile, target_dir):
"""Safer version of ZipFile.extractall -- does not allow absolute or upwards-relative paths"""
for zipinfo in zipfile.infolist():
# skip absolute or upwards-relative files
if zipinfo.filename.startswith(('/', '..')):
warnings.warn('Skipping potentially unsafe file: ' + zipinfo.filename, RuntimeWarning)
continue
# target_dir is base directory; extract will create subpaths as necessary
zipfile.extract(zipinfo, target_dir)
def _write_data_to_folder(zipfile, labels, folder, isTrainingset):
print("Writing data to folder\n")
for i, (enrty, label) in enumerate(zip(zipfile.infolist()[1:],
labels)):
with zipfile.open(enrty) as file:
img = Image.open(file)
dest = os.path.join(folder, str(label))
make_folder_if_not_exists(dest)
if isTrainingset == 1:
img.save(os.path.join(dest, str(i) + '.png'))
else:
img.save(os.path.join(dest, str(i) + '.jpg'))
def _write_data_to_folder(zipfile, filenames, labels, folder, start_index,
isTest):
print("Writing data\n")
sorted_labels = [None] * len(labels)
if isTest == 1:
for i in range(len(zipfile.infolist())):
entry = zipfile.infolist()[i]
if "IRHT_P_009793.tif" in entry.filename:
zipfile.infolist().remove(entry)
break
zip_infolist = zipfile.infolist()[1:]
for i in range(len(zip_infolist)):
entry = zip_infolist[i]
entry_index_infilenames = filenames.index(
entry.filename[start_index:])
sorted_labels[i] = labels[entry_index_infilenames]
for i, (enrty,
label) in enumerate(zip(zipfile.infolist()[1:],
sorted_labels)):
with zipfile.open(enrty) as file:
img = Image.open(file)
dest = os.path.join(folder, str(label))
make_folder_if_not_exists(dest)
img.save(os.path.join(dest,
str(i) + '.png'),
"PNG",
quality=100)
def read_packs(zipfile):
all_packs = collections.defaultdict(list)
packs_by_gameid = collections.defaultdict(
lambda: collections.defaultdict(set))
for info in zipfile.infolist():
packname = info.filename.split('/', 1)[0]
all_packs[packname].append(info)
if info.filename.endswith('/rules.txt') and info.filename.count(
'/') == 1:
with zipfile.open(info) as f:
pack_game_ids, res = _parse_rules_txt(f)
if pack_game_ids is not None:
for _id in pack_game_ids:
packs_by_gameid[_id][res].add(packname)
logger.debug('%d packs found in zip', len(all_packs))
return packs_by_gameid, all_packs
def xmls(mimetype, data):
if mimetype == 'application/zip':
print('processing zipfile')
zio = zipfile.ZipFile(data)
zil = zipfile.infolist()
for fnm in zil:
xdata = zio.read(fnm)
with magic.Magic(flags=magic.MAGIC_MIME_TYPE) as m:
mt = m.id_buffer(xdata)
if mt == 'application/xml':
yield xdata
elif mimetype == 'application/xml' or mimetype == 'text/xml':
print('processing single xml')
yield data
else:
return None
def _yamlfiles(zipfile, has_subdirs=False):
"""Helper to get a yaml file (according to POLICYD_VALID_EXTS extensions)
and the infolist item from a zipfile.
If the `has_subdirs` param is True, the the only yaml files that have a
directory component are read, and then first part of the directory
component is kept, along with the filename in the name. e.g. an entry with
a filename of:
compute/someotherdir/override.yaml
is returned as:
compute/override, yaml, override.yaml, <ZipInfo object>
This is to help with the special, additional, processing that the dashboard
charm requires.
:param zipfile: the zipfile to read zipinfo items from
:type zipfile: zipfile.ZipFile
:param has_subdirs: Keep first level of subdirectories in yaml file.
:type has_subdirs: bool
:returns: generator of (name, ext, filename, info item) for each
self-identified yaml file.
:rtype: List[(str, str, str, zipfile.ZipInfo)]
"""
files = []
for infolist_item in zipfile.infolist():
try:
if infolist_item.is_dir():
continue
except AttributeError:
# fallback to "old" way to determine dir entry for pre-py36
if infolist_item.filename.endswith('/'):
continue
_dir, name_ext = os.path.split(infolist_item.filename)
name, ext = os.path.splitext(name_ext)
if has_subdirs and _dir != "":
name = os.path.join(_dir.split(os.path.sep)[0], name)
ext = ext.lower()
if ext and ext in POLICYD_VALID_EXTS:
files.append((name, ext, name_ext, infolist_item))
return files
def _yamlfiles(zipfile):
"""Helper to get a yaml file (according to POLICYD_VALID_EXTS extensions)
and the infolist item from a zipfile.
:param zipfile: the zipfile to read zipinfo items from
:type zipfile: zipfile.ZipFile
:returns: generator of (name, ext, filename, info item) for each self-identified
yaml file.
:rtype: List[(str, str, str, zipfile.ZipInfo)]
"""
l = []
for infolist_item in zipfile.infolist():
if infolist_item.is_dir():
continue
_, name_ext = os.path.split(infolist_item.filename)
name, ext = os.path.splitext(name_ext)
ext = ext.lower()
if ext and ext in POLICYD_VALID_EXTS:
l.append((name, ext, name_ext, infolist_item))
return l
def decrypt_sound_found_in_memory(self, in_file_path):
with open(in_file_path, 'rb') as in_file:
file_size = struct.unpack('<Q',
in_file.read(struct.calcsize('<Q')))[0]
iv = in_file.read(16)
aes = AES.new(self.key, AES.MODE_CBC, iv)
out_file = tempfile.NamedTemporaryFile(delete=False)
while True:
data = in_file.read(self.size_chunk)
n = len(data)
if n == 0:
break
decode = aes.decrypt(data)
n = len(decode)
if file_size > n:
out_file.write(decode)
else:
out_file.write(
decode[:file_size]) # <- remove padding on last block
file_size -= n
temp_path = out_file.name[:out_file.name.rfind('\\')]
import zipfile
zipfile = zipfile.ZipFile(out_file)
zipfile.extractall(temp_path)
zipfile.close()
info_zipfile = zipfile.infolist()
if len(info_zipfile) < 2:
info_file_path = info_zipfile[0].filename.replace('/', '\\')
print(info_file_path)
sf_path = temp_path + '\\' + info_file_path
print(sf_path)
return sf_path
## Topic: Assignment 10
## Name: Gabriel Groover
## Section time: 330-445
## Grading group: 2
import zipfile
import pandas as pd
#Create zipfile from the stocks zip
zipfile = zipfile.ZipFile('Stocks.zip')
dfList = []
#Iterate through each of the items in the zipfile
#File contains all information about a single file
for file in zipfile.infolist():
#If the file is a csv, open it and read as csv into dataframe
if file.filename[-4:] == '.csv':
df = pd.read_csv(zipfile.open(file))
#add column for ticker symbol and add dataframe to list
df['Stock'] = file.filename[7:-4]
dfList.append(df)
#Combine all frames in list
stocks = pd.concat(dfList)
# 1. Find the mean for the Open, High, Low, and Close entries for all of the records taken as a single group.
#Get stock columns with correct values and take means for those columns
print(stocks[['Open', 'Close', 'High', 'Low']].mean())
'''
def get_zip_contents(zipfile):
ret = {}
for info in zipfile.infolist():
ret[info.filename] = info
return ret
print(" 1. Checking filename.............. ", end="", flush=True)
if not FLAGS.zipfile.endswith('.zip'):
raise ValidationException(
'Competition bundle must end with ".zip"')
print(checkmark)
with zipfile.ZipFile(FLAGS.zipfile) as zipfile:
if FLAGS.task == "segmentation":
print(" 2. Checking directory structure... ",
end="",
flush=True)
directories = [
folder.filename for folder in zipfile.infolist()
if folder.filename.endswith("/")
]
if "sequences/" not in directories:
raise ValidationException(
'Directory "sequences" missing inside zip file.')
for sequence in range(11, 22):
sequence_directory = "sequences/{}/".format(sequence)
if sequence_directory not in directories:
raise ValidationException(
'Directory "{}" missing inside zip file.'.format(
sequence_directory))
predictions_directory = sequence_directory + "predictions/"
if predictions_directory not in directories:
raise ValidationException(
