def extract(self, path, sel_files=None):
count = 0.0
for u, files in self.urls:
if sel_files is not None:
files = set(files) & sel_files
count += len(files)
i = 0
for u, files in self.urls:
if sel_files is not None:
files = set(files) & sel_files
for item in files:
mypath = os.path.join(path, item)
if os.path.exists(mypath) and is_archive(mypath):
progress.update(i / count, 'Extracting "%s"...' % item)
with tempfile.TemporaryDirectory() as tempdir:
# Extract to a temporary directory and then move the result
# to final destination to avoid "Do you want to overwrite?" questions.
extract_archive(mypath, tempdir)
if self.ignore_subpath:
for sub_path, dirs, files in os.walk(tempdir):
for name in files:
shutil.move(os.path.join(sub_path, name), ipath(os.path.join(path, name)))
else:
movetree(tempdir, path, ifix=True)
i += 1
def movetree(src, dest, ifix=False):
if ifix:
dest = ipath(dest)
if not os.path.isdir(dest):
os.makedirs(dest)
if ifix:
siblings = os.listdir(dest)
l_siblings = [s.lower() for s in siblings]
for item in os.listdir(src):
spath = os.path.join(src, item)
dpath = os.path.join(dest, item)
if ifix and not os.path.exists(dpath):
if item.lower() in l_siblings:
l_item = siblings[l_siblings.index(item.lower())]
logging.warning('Changing path "%s" to "%s" to avoid case problems...', dpath, os.path.join(dest, l_item))
dpath = os.path.join(dest, l_item)
else:
siblings.append(item)
l_siblings.append(item.lower())
if os.path.isdir(spath):
movetree(spath, dpath)
else:
if os.path.exists(dpath):
os.unlink(dpath)
shutil.move(spath, dpath)
def extract(self, path, sel_files=None):
count = 0.0
for u, files in self.urls:
if sel_files is not None:
files = set(files) & sel_files
count += len(files)
i = 0
for u, files in self.urls:
if sel_files is not None:
files = set(files) & sel_files
for item in files:
mypath = os.path.join(path, item)
if os.path.exists(mypath) and is_archive(mypath):
progress.update(i / count, 'Extracting "%s"...' % item)
with tempfile.TemporaryDirectory() as tempdir:
# Extract to a temporary directory and then move the result
# to final destination to avoid "Do you want to overwrite?" questions.
extract_archive(mypath, tempdir)
if self.ignore_subpath:
for sub_path, dirs, files in os.walk(tempdir):
for name in files:
shutil.move(
os.path.join(sub_path, name),
ipath(os.path.join(path, name)))
else:
movetree(tempdir, path, ifix=True)
i += 1
def movetree(src, dest, ifix=False):
if ifix:
dest = ipath(dest)
if not os.path.isdir(dest):
os.makedirs(dest)
if ifix:
siblings = os.listdir(dest)
l_siblings = [s.lower() for s in siblings]
for item in os.listdir(src):
spath = os.path.join(src, item)
dpath = os.path.join(dest, item)
if ifix and not os.path.exists(dpath):
if item.lower() in l_siblings:
l_item = siblings[l_siblings.index(item.lower())]
logging.warning(
'Changing path "%s" to "%s" to avoid case problems...',
dpath, os.path.join(dest, l_item))
dpath = os.path.join(dest, l_item)
else:
siblings.append(item)
l_siblings.append(item.lower())
if os.path.isdir(spath):
movetree(spath, dpath)
else:
if os.path.exists(dpath):
os.unlink(dpath)
shutil.move(spath, dpath)
def check_hashes(self, path):
alright = True
for algo, filepath, chksum in self.hashes:
try:
mysum = gen_hash(ipath(os.path.join(path, filepath)), algo)
except:
logging.exception('Failed to computed checksum for "%s" with algorithm "%s"!', filepath, algo)
continue
if mysum != chksum.lower():
alright = False
logging.warning('File "%s" has checksum "%s" but should have "%s"! Used algorithm: %s', filepath, mysum, chksum, algo)
return alright
def check_hashes(self, path):
alright = True
for algo, filepath, chksum in self.hashes:
try:
mysum = gen_hash(ipath(os.path.join(path, filepath)), algo)
except:
logging.exception(
'Failed to computed checksum for "%s" with algorithm "%s"!',
filepath, algo)
continue
if mysum != chksum.lower():
alright = False
logging.warning(
'File "%s" has checksum "%s" but should have "%s"! Used algorithm: %s',
filepath, mysum, chksum, algo)
return alright