def sevenZipCrackAction(self):
progressBarValue = 0
tested = 0
for password in open(self.passwordlist):
try:
app.processEvents() # Process Events when 0fCrack is Cracking ...
except :
pass
password = password.strip("\n")
file = SevenZipFile(self.file , mode = "r" , password = password)
try:
file.testzip()
return password
except LZMAError:
tested += 1
if self.guimode:
self.progressBar.setProperty("value" , (tested * 100) / self.n)
else:
self.bar.update(tested)
except EOFError:
tested += 1
if self.guimode:
self.progressBar.setProperty("value" , (tested * 100) / self.n)
else:
self.bar.update(tested)
return False
def DecryptRuShiWoWen(ciphertext):
data = b''
for i in ciphertext:
data += bytes([ruShiWoWen.index(i)])
cryptor = AES.new(KEY, AES.MODE_CBC, IV)
fsevenZip=SevenZipFile(BytesIO(cryptor.decrypt(data)))
zipContent = fsevenZip.readall()['default'].read()
return zipContent
def extract(archive_name: str, manage_folder: bool = True) -> None:
"""
Extract `archive_name` and put it inside a folder
if there are multiple files inside.
:param archive_name: path to archive
:param manage_folder: check if there is root folder in archive:
if there is one, do not create extra folder,
if there are just files inside, put them into folder.
If param is set to `False`, extract "as is".
:return:
"""
if archive_name.endswith('.zip'):
archive = ZipFile(archive_name)
elif archive_name.endswith('.7z'):
archive = SevenZipFile(archive_name)
else:
try:
archive = tarfile.open(archive_name)
except Exception:
raise NotImplementedError(f'Can\'t extract {archive_name}')
name = os.path.dirname(archive_name)
if manage_folder and not contains_dir(archive):
name = remove_extension(archive_name)
os.mkdir(name)
archive.extractall(path=name)
archive.close()
def extract_py7zr(archive_path, theme, layer_name, target_dir):
"""
Extraction using py7zr package
"""
with SevenZipFile(archive_path, 'r') as archive:
# Extracts the compressed shapefile
to_extract = []
for f in archive.getnames():
match = _test_file(f)
if match and theme == match.group(1) \
and layer_name == match.group(2):
to_extract.append(f)
if len(to_extract) == 0: # layer not found
return None
archive.extract(targets=to_extract, path=target_dir)
# move extracted files to the root folder and deletes created folders
for f in to_extract:
fname = os.path.basename(f)
os.replace(os.path.join(target_dir, f),
os.path.join(target_dir, fname))
dirname = os.path.splitext(os.path.basename(archive_path))[0]
# causes problems with parallel extractions
#shutil.rmtree(os.path.join(target_dir, dirname))
_clean_empty_folders(os.path.join(target_dir, dirname))
return os.path.join(target_dir, layer_name + '.shp')
def download_archive_and_add_subtitle_files(self, link, language, video,
fps, num_cds):
logger.info('Downloading subtitle %r', link)
cache_key = sha1(link.encode("utf-8")).digest()
request = region.get(cache_key)
if request is NO_VALUE:
request = self.session.get(
link, headers={'Referer': 'https://subsunacs.net/search.php'})
request.raise_for_status()
region.set(cache_key, request)
else:
logger.info('Cache file: %s',
codecs.encode(cache_key, 'hex_codec').decode('utf-8'))
try:
archive_stream = io.BytesIO(request.content)
if is_rarfile(archive_stream):
return self.process_archive_subtitle_files(
RarFile(archive_stream), language, video, link, fps,
num_cds)
elif is_zipfile(archive_stream):
return self.process_archive_subtitle_files(
ZipFile(archive_stream), language, video, link, fps,
num_cds)
elif archive_stream.seek(0) == 0 and is_7zfile(archive_stream):
return self.process_archive_subtitle_files(
SevenZipFile(archive_stream), language, video, link, fps,
num_cds)
except:
pass
logger.error('Ignore unsupported archive %r', request.headers)
region.delete(cache_key)
return []
def encrypt(self): #zip all the directories from data.xml into a passworded archive
fileNumber = 0
zippedFileNumber = 0
errors = 0
errortext = []
tree = etree.parse("data.xml")
########### WARNING #############
#To avoid having the password stored in plaintext, the zip archived will be locked with the password
#in its HASHED FORM. The algorithm is SHA3_512.
#to decrypt the archive, one will need to hash his password and use the output.
#or just read it in data.xml if the root password is known.
#####################################
pw = tree.find("password").text #only way to not have the password in clear text here, or use a reversible algorithm.
try: #py7zr doesn't support 'append', so next directory in archive will overwrite the previous.
mkdir(tree.find('storage').text + '/temp') #solution is to create an archive for each directory, then zip them all in the final .7z
except FileExistsError: #this results in the final .7z archive being passworded and containing a number of passworded .7z.
pass
for i in tree.getroot(): # locations MUST be directories, not single files, otherwise will not be backed up.
if i.tag.startswith('loc'):
for target in i.getiterator():
print(target.text)
with SevenZipFile(tree.find('storage').text + '/temp/' + target.text.replace('/', '.') + '.7z',
'w', password=pw) as archive:
try:
fileNumber += 1
archive.writeall(target.text, 'base')
zippedFileNumber += 1
except Exception as e:
#print(e)
errors += 1 #not sure what errors to expect with py7zr.
errortext.append(str(e))
pass
with SevenZipFile(tree.find('storage').text + '/encrypted_data.7z', 'w', password=pw) as archive:
archive.writeall(tree.find('storage').text + '/temp/', 'base')
#TODO : manage the various exeption types to abort or ignore.
if errors == 0:
self.sendmail("encrypt", 0)
else:
text = ""
for i in errortext:
text = text + str(i) + '<br>'
text = MIMEText(text, 'html')
self.sendmail("encrypt", text)
if zippedFileNumber / fileNumber > 0.9: # if more than 90% of the directories were backed up in the encrypted archive ? Maybe set max number of errors, too ?
print("wiping")
class SevenZipArchive(BaseArchive):
def __init__(self, file):
self._archive = SevenZipFile(file, mode='r')
def list(self, *args, **kwargs):
self._archive.printdir(*args, **kwargs)
def filenames(self):
return self._archive.getnames()
def namelist(self):
return self._archive.getnames()
def close(self):
return self._archive.close()
def extract(self, file):
return self._archive.extract(file)
def is_encrypted(self):
return self._archive.password_protected
def extractall(self, file):
return self._archive.extractall(file)
def iter_7zip(
file: Union[BinaryIO, str, PathLike],
mode: str = "rb",
encoding: Optional[str] = None,
errors: Optional[str] = None,
newline: Optional[str] = None,
password: Optional[str] = None,
) -> Iterator[Tuple[str, IO]]:
from py7zr import SevenZipFile
encoding = _check_arguments(mode, encoding)
newmode = _stripmode(mode)
with SevenZipFile(file, mode=newmode) as zf:
for fname, bf in zf.readall().items():
yield fname, wrap_text(bf, mode, encoding, errors, newline)
def run(self):
#Extract 7z
for file in os.listdir(self.localPath):
path7Z = self.localPath+file
if self.checkFileCorrupted(path7Z):
with SevenZipFile(path7Z, 'r') as zipFile:
allFiles = zipFile.getnames()
for f in allFiles:
if self.checkFileExtension(f,'.0'):
fileName,fileExtension = self.getFileInfo(f)
zipFile.extract(localPathNew+fileName)
else:
print("Check 1 - Corrupted compression - excluded file: "+file)
print ("--- Extract 7z: %s seconds ---" % (time.time() - startTime))
#Read and create parquet
for fileName in os.listdir(self.localPathNew):
csvFilePath = self.localPathNew+fileName+'/'+fileName+'.0'
parquetFilePath = self.localPathNew+fileName+'.parquet'
dfConsumtions = spark.read.csv(csvFilePath,header=False, sep=';')
if self.checkFirstColumn(dfConsumtions):
dfConsumtions2 = dfConsumtions.withColumn('file_name',lit(fileName))
dfConsumtions2.write.parquet(parquetFilePath,mode='overwrite')
else:
print("Check 2 - Length of firt column - excluded file: "+fileName)
print ("--- Read and create parquet: %s seconds ---" % (time.time() - startTime))
#Upload to S3
s3_client = boto3.client('s3',aws_access_key_id=ACCESS_KEY,aws_secret_access_key=SECRET_KEY)
bucket = 'consumos-ss'
folder = 'consumos/'
for fileName in os.listdir(self.localPathNew):
if self.checkFileExtension(fileName,'.parquet'):
parquetFilePath2 = self.localPathNew+fileName+'/'
for fileNameParquet in os.listdir(parquetFilePath2):
if self.checkFileExtension(fileNameParquet,'.parquet'):
finalLocalPath = parquetFilePath2+fileNameParquet
folderObject = folder+fileNameParquet
try:
response = s3_client.upload_file(finalLocalPath,bucket,folderObject)
except ClientError as e:
logging.error(e)
print ("--- Upload to S3: %s seconds ---" % (time.time() - startTime))
return ("--- %s seconds ---" % (time.time() - startTime))
def load_qider(path, imgsize):
""" Loads and normalizes the images from the QIDER dataset. """
data, labels = [], []
for exp in EXPRESSIONS:
exp_path = os.path.join(path, exp)
if not os.path.isdir(exp_path):
with SevenZipFile(os.path.join(path, exp + ".7z"), 'r') as zip:
zip.extractall(path)
for img in os.listdir(exp_path):
data.append(
np.asarray(
Image.open(os.path.join(exp_path, img)).resize(imgsize)) /
255)
labels.append(EXPRESSIONS[exp])
return np.asarray(data).reshape(len(data), imgsize[0], imgsize[1],
1), np.asarray(labels)
def extract():
# extract tar files
for (dir_path1, dir_names, files1) in os.walk(tar_path):
file_list = set([])
target_path1 = dir_path1.replace('tar', 'raw')
try:
os.mkdir(target_path1)
except Exception as e1:
print(e1)
# get list of file
for file_name1 in files1:
print(file_name1)
file_list.add(file_name1[:-4])
print(file_list)
# extraction
for file in file_list:
print(dir_path1 + '/' + file)
with multivolumefile.open(dir_path1 + '/' + file,
mode='rb') as target_archive:
with SevenZipFile(target_archive, 'r') as archive:
archive.extractall(target_path1)
def extract(self):
if os.name == 'nt':
rarfile.UNRAR_TOOL = self.WINDOWS_OS_UNRAR_PATH
print("\n******************** EXTRACTED files ********************")
count = 1
for format in self.SUPPORTED_FORMATS:
for file in glob(format):
try:
if format == '*.zip':
with ZipFile(file, 'r') as zipObj:
zipObj.extractall(
os.path.normpath(
os.getcwd() + os.sep + '..' + os.sep +
self.DESTINITION_DIRECTORY + os.sep +
str(self.INITIAL_DIRECTORY_NAME)))
elif format == '*.rar':
with rarfile.RarFile(file, 'r') as rarObj:
rarObj.extractall(
os.path.normpath(
os.getcwd() + os.sep + '..' + os.sep +
self.DESTINITION_DIRECTORY + os.sep +
str(self.INITIAL_DIRECTORY_NAME)))
elif format == '*.7z':
with SevenZipFile(file, 'r') as sevenZObj:
sevenZObj.extractall(
os.path.normpath(
os.getcwd() + os.sep + '..' + os.sep +
self.DESTINITION_DIRECTORY + os.sep +
str(self.INITIAL_DIRECTORY_NAME)))
print(
f'{count}. FILE: {file} =====> {str(self.INITIAL_DIRECTORY_NAME)}\n----------------------------------------------------------------------------------------'
)
self.INITIAL_DIRECTORY_NAME += 1
count += 1
except Exception as ex:
self.EXCEPTIONS[
f'File: {str(file)}'] = f'Exception type: {sys.exc_info()[0]}, {ex}'
def test_register_archive_format(tmp_path):
tmp_path.joinpath('src').mkdir()
tmp_path.joinpath('tgt').mkdir()
# Prepare test data
py7zr.unpack_7zarchive(os.path.join(testdata_path, 'test_1.7z'), path=tmp_path.joinpath('src'))
#
shutil.register_archive_format('7zip', pack_7zarchive, description='7zip archive')
shutil.make_archive(str(tmp_path.joinpath('target')), '7zip', str(tmp_path.joinpath('src')))
# check result
archive = SevenZipFile(tmp_path.joinpath('target.7z'), 'r')
archive.extractall(path=tmp_path.joinpath('tgt'))
archive.close()
m = hashlib.sha256()
m.update((tmp_path / 'tgt' / 'setup.py').open('rb').read())
assert m.digest() == binascii.unhexlify('b916eed2a4ee4e48c51a2b51d07d450de0be4dbb83d20e67f6fd166ff7921e49')
m = hashlib.sha256()
m.update((tmp_path / 'tgt' / 'scripts' / 'py7zr').open('rb').read())
assert m.digest() == binascii.unhexlify('b0385e71d6a07eb692f5fb9798e9d33aaf87be7dfff936fd2473eab2a593d4fd')
def test_register_archive_format(tmp_path):
tmp_path.joinpath("src").mkdir()
tmp_path.joinpath("tgt").mkdir()
# Prepare test data
py7zr.unpack_7zarchive(os.path.join(testdata_path, "test_1.7z"), path=tmp_path.joinpath("src"))
#
shutil.register_archive_format("7zip", pack_7zarchive, description="7zip archive")
shutil.make_archive(str(tmp_path.joinpath("target")), "7zip", str(tmp_path.joinpath("src")))
# check result
archive = SevenZipFile(tmp_path.joinpath("target.7z"), "r")
archive.extractall(path=tmp_path.joinpath("tgt"))
archive.close()
m = hashlib.sha256()
m.update((tmp_path / "tgt" / "setup.py").open("rb").read())
assert m.digest() == binascii.unhexlify("b916eed2a4ee4e48c51a2b51d07d450de0be4dbb83d20e67f6fd166ff7921e49")
m = hashlib.sha256()
m.update((tmp_path / "tgt" / "scripts" / "py7zr").open("rb").read())
assert m.digest() == binascii.unhexlify("b0385e71d6a07eb692f5fb9798e9d33aaf87be7dfff936fd2473eab2a593d4fd")
def _download_dataset(self):
if not osp.isdir(self._dataset_root):
os.mkdir(self._dataset_root)
if self._verbose >= 1:
print('Download files for the dataset...')
infos = FILES_INFOS[self._subset]
# Download archives files
for name, info in infos.items():
filename, url, hash_ = info['filename'], info['url'], info['hash']
filepath = osp.join(self._dataset_root, filename)
if not osp.isfile(filepath):
if self._verbose >= 1:
print(f'Download file "{filename}" from url "{url}"...')
if osp.exists(filepath):
raise RuntimeError(
f'Object "{filepath}" already exists but it\'s not a file.'
)
download_url(url,
self._dataset_root,
filename,
hash_value=hash_,
hash_type='md5')
# Extract audio files from archives
for name, info in infos.items():
filename = info['filename']
filepath = osp.join(self._dataset_root, filename)
extension = filename.split('.')[-1]
if extension == '7z':
extracted_path = osp.join(self._dataset_root, self._subset)
if not osp.isdir(extracted_path):
if self._verbose >= 1:
print(f'Extract archive file "{filename}"...')
archive_file = SevenZipFile(filepath)
archive_file.extractall(self._dataset_root)
archive_file.close()
def un7zipFileToDir(sevenZipFilePath, dirPath):
with SevenZipFile(sevenZipFilePath, mode='r') as sevenZipObj:
sevenZipObj.extractall(dirPath)
session = requests.Session()
session.verify = False
protect = RemoteCKAN('https://data-protect-slr.univ-grenoble-alpes.fr/',
apikey='xxxxxxxxxxxxxxxxxx',
user_agent='xxxxxxxxxxxx',
session=session)
# Specify the zip file path + explore and get the files to upload
zip_path = '/data/ckan/storage/CNRS/'
zip_name = 'data_1go.7z'
# Check the metadata of the uploaded file
with SevenZipFile(zip_path+zip_name,'r') as myzip:
zipped_files = myzip.getnames()
print("Files to Upload:",zipped_files)
for fichier in zipped_files:
filename, file_extension = os.path.splitext(zip_path+fichier)
ressource_dict = {
'package_id':'dataset_5',
'upload': open(zip_path+fichier,'rb'), # for local file
'name': fichier,
'format':file_extension.replace('.',''),
'description':'Upload with 7-zip file',
'notes': 'A long description of my dataset',
'url ':''
}
import_res = protect.action.resource_create(**ressource_dict)
# Generate UniStore entries
for skin in files:
print(skin)
# Skip Wood UI for now
if (getTheme(skin) == "Wood UI"):
continue
info = {}
updated = datetime.utcfromtimestamp(0)
inFolder = False
skinName = skin[skin.rfind("/") + 1:skin.rfind(".")]
if skin[-2:] == "7z":
with SevenZipFile(skin) as a:
updated = lastUpdated(a)
inFolder = skinName in a.getnames()
else:
updated = datetime.utcfromtimestamp(
int(
git.Repo(".").git.log(
["-n1", "--pretty=format:%ct", "--", skin]) or 0))
created = datetime.utcfromtimestamp(
int(
git.Repo(".").git.log(["--pretty=format:%ct", "--", skin
]).split("\n")[-1] or 0))
if path.exists(
path.join(skin[:skin.rfind("/")], "meta", skinName, "info.json")):
def extract(archive: SevenZipFile = archive,
info: FileInfo = info):
archive.reset()
return archive.read(info.filename).get(info.filename).read()
def open_archive(path_or_stream, mode='r'):
from py7zr import SevenZipFile
return SevenZipFile(path_or_stream, mode=mode)
def __init__(self, file):
self._archive = SevenZipFile(file, mode='r')
def un_archive(self):
with SevenZipFile(self.archiver, mode='r', password=self.pwd) as z:
z.extractall(path=self.path)