def update_progress(pi):
total, items, title = task.get_progress()
text = []
for item in items.values():
text.append('%3d%% %s' % (item[0] * 100, item[1]))
progress.update(total, '\n'.join(text))
def extract(self, path, sel_files=None):
count = 0.0
for u, files in self.urls:
if sel_files is not None:
files = set(files) & sel_files
count += len(files)
i = 0
for u, files in self.urls:
if sel_files is not None:
files = set(files) & sel_files
for item in files:
mypath = os.path.join(path, item)
if os.path.exists(mypath) and is_archive(mypath):
progress.update(i / count, 'Extracting "%s"...' % item)
with tempfile.TemporaryDirectory() as tempdir:
# Extract to a temporary directory and then move the result
# to final destination to avoid "Do you want to overwrite?" questions.
extract_archive(mypath, tempdir)
if self.ignore_subpath:
for sub_path, dirs, files in os.walk(tempdir):
for name in files:
shutil.move(os.path.join(sub_path, name), ipath(os.path.join(path, name)))
else:
movetree(tempdir, path, ifix=True)
i += 1
def extract(self, path, sel_files=None):
count = 0.0
for u, files in self.urls:
if sel_files is not None:
files = set(files) & sel_files
count += len(files)
i = 0
for u, files in self.urls:
if sel_files is not None:
files = set(files) & sel_files
for item in files:
mypath = os.path.join(path, item)
if os.path.exists(mypath) and is_archive(mypath):
progress.update(i / count, 'Extracting "%s"...' % item)
with tempfile.TemporaryDirectory() as tempdir:
# Extract to a temporary directory and then move the result
# to final destination to avoid "Do you want to overwrite?" questions.
extract_archive(mypath, tempdir)
if self.ignore_subpath:
for sub_path, dirs, files in os.walk(tempdir):
for name in files:
shutil.move(
os.path.join(sub_path, name),
ipath(os.path.join(path, name)))
else:
movetree(tempdir, path, ifix=True)
i += 1
def write(self):
self.pack('4siii', b'VPVP', 0, 0, 0)
toc = []
self.write_dir(self._files, toc)
diroffset = self._file.tell()
dirs = 0
files = 0
progress.update(0.99, 'Writing TOC...')
for item in toc:
self.pack('ii32si', item['offset'], item['size'],
item['name'].encode('utf8'), item['timestamp'])
if item['timestamp'] == 0:
dirs += 1
else:
files += 1
self._file.seek(0)
self.pack('4siii', b'VPVP', 2, diroffset, len(toc))
self._file.close()
logging.info('Wrote {0} files and {1} directories in "{2}".'.format(
files, dirs, self._file.name))
def write_file(self, name, content, toc):
progress.update(self._written / float(self._count),
'Packing "%s"...' % name)
self._written += 1
if isinstance(content, str):
content = open(content, 'rb')
opened = True
else:
opened = False
offset = self._file.tell()
shutil.copyfileobj(content, self._file)
size = self._file.tell() - offset
if size == 0:
raise EmptyFileException(content)
if opened:
content.close()
toc.append({
'name': name,
'offset': offset,
'size': size,
'timestamp': int(time())
})
def update_progress(pi):
total, items, title = task.get_progress()
text = []
for item in items.values():
text.append('%3d%% %s' % (item[0] * 100, item[1]))
progress.update(total, '\n'.join(text))
def write(self, fn, gen_out=False):
assert not gen_out
if not self._list:
return False
count = len(self._list) + len(self._added)
done = 0.0
for item in self._list:
progress.update(done / count, 'Processing "%s"...' % item)
done += 1
if item in self._modified:
shutil.copyfile(self._modified[item], os.path.join(fn, item))
elif item not in self._deleted:
shutil.copyfile(os.path.join(self._handle, item),
os.path.join(fn, item))
for item, path in self._added.items():
progress.update(done / count, 'Processing "%s"...' % item)
done += 1
shutil.copyfile(path, os.path.join(fn, item))
return True
def execute_rename(self, path):
count = float(len(self.rename))
i = 0
for src, dest in self.rename:
logging.info('Moving "%s" to "%s"...', src, dest)
progress.update(i / count, 'Moving "%s" to "%s"...' % (src, dest))
if os.path.exists(src):
shutil.move(src, dest)
else:
logging.warning('"%s" not found!', src)
i += 1
def execute_rename(self, path):
count = float(len(self.rename))
i = 0
for src, dest in self.rename:
logging.info('Moving "%s" to "%s"...', src, dest)
progress.update(i / count, 'Moving "%s" to "%s"...' % (src, dest))
if os.path.exists(src):
shutil.move(src, dest)
else:
logging.warning('"%s" not found!', src)
i += 1
def execute_del(self, path):
count = float(len(self.delete))
for i, item in enumerate(self.delete):
logging.info('Deleting "%s"...', item)
progress.update(i / count, 'Deleting "%s"...' % item)
item = os.path.join(path, item)
if os.path.isdir(item):
shutil.rmtree(item)
elif os.path.exists(item):
os.unlink(item)
else:
logging.warning('"%s" not found!', item)
def execute_del(self, path):
count = float(len(self.delete))
for i, item in enumerate(self.delete):
logging.info('Deleting "%s"...', item)
progress.update(i / count, 'Deleting "%s"...' % item)
item = os.path.join(path, item)
if os.path.isdir(item):
shutil.rmtree(item)
elif os.path.exists(item):
os.unlink(item)
else:
logging.warning('"%s" not found!', item)
def work(self, item):
id_, links, name, archive, tstamp = item
with tempfile.TemporaryDirectory() as dest:
if self.dl_path is None:
base_path = dest
else:
base_path = os.path.join(self.dl_path, self.dl_slug)
f_name = os.path.basename(name)
path = os.path.join(base_path, f_name)
idx = 1
while os.path.exists(path):
path = os.path.join(base_path, str(idx) + '_' + f_name)
idx += 1
res = self._download(links, path, tstamp)
for i, link in reversed(list(enumerate(links))):
for pref in self.rem_prefixes:
if link.startswith(pref):
del links[i]
if res == 304:
# Nothing changed.
self.post((id_, 'CACHE', None, 0))
elif res:
logging.info('Inspecting "%s"...', name)
progress.update(0.999, 'Inspecting "%s"...' % name)
csum, content = self._inspect_file(id_, archive, dest, path)
if csum != 'FAILED':
if self.dl_mirror is not None:
links.append(util.pjoin(self.dl_mirror, self.dl_slug, os.path.basename(path)))
self.post((id_, csum, content, os.path.getsize(path)))
else:
os.unlink(path)
self.post((id_, 'FAILED', None, 0))
else:
# None of the links worked!
self.post((id_, 'FAILED', None, 0))
def cleanup(self, path, sel_files=None):
count = 0.0
for u, files in self.urls:
if sel_files is not None:
files = set(files) & sel_files
count += len(files)
i = 0
for u, files in self.urls:
if sel_files is not None:
files = set(files) & sel_files
for item in files:
mypath = os.path.join(path, item)
if os.path.exists(mypath) and is_archive(mypath):
# Only remove the archives...
progress.update(i / count, 'Removing "%s"...' % item)
os.unlink(mypath)
i += 1
def cleanup(self, path, sel_files=None):
count = 0.0
for u, files in self.urls:
if sel_files is not None:
files = set(files) & sel_files
count += len(files)
i = 0
for u, files in self.urls:
if sel_files is not None:
files = set(files) & sel_files
for item in files:
mypath = os.path.join(path, item)
if os.path.exists(mypath) and is_archive(mypath):
# Only remove the archives...
progress.update(i / count, 'Removing "%s"...' % item)
os.unlink(mypath)
i += 1
def write(self, fn, gen_out=False):
if not self._list:
return False
if gen_out:
fd, fn = tempfile.mkstemp(suffix='.7z', dir=fn)
os.close(fd)
with tempfile.TemporaryDirectory() as path:
progress.start_task(0, 0.5)
super(SevenAdapter, self).write(path)
progress.finish_task()
progress.update(0.5, 'Compressing...')
subprocess.check_call(
['7z', 'a', os.path.abspath(fn), '.'], cwd=path)
if gen_out:
return fn
else:
return True
def work(self, item):
id_, links, name, archive, tstamp = item
with tempfile.TemporaryDirectory() as dest:
if self.dl_path is None:
base_path = dest
else:
base_path = self.dl_path
f_name = os.path.basename(name)
path = os.path.join(base_path, f_name)
idx = 1
while os.path.exists(path):
path = os.path.join(base_path, str(idx) + '_' + f_name)
idx += 1
res = self._download(links, path, tstamp)
if res == 304:
# Nothing changed.
self.post((id_, 'CACHE', None, 0))
elif res:
logging.info('Inspecting "%s"...', name)
progress.update(1, 'Inspecting "%s"...' % name)
csum, content = self._inspect_file(id_, archive, dest, path)
if csum != 'FAILED':
if self.dl_mirror is not None:
links.append(
util.pjoin(self.dl_mirror, os.path.basename(path)))
self.post((id_, csum, content, os.path.getsize(path)))
else:
self.post((id_, 'FAILED', None, 0))
else:
# None of the links worked!
self.post((id_, 'FAILED', None, 0))