def fetch_amendements(chambre: Optional[str], num: Optional[int],
progress: bool, settings: Dict[str, Any]) -> None:
an_ids = repository.list_opendata_dossiers()
if progress:
bar = ProgressBar(total=len(an_ids))
random.shuffle(an_ids)
for an_id in an_ids:
dossier_ref = repository.get_opendata_dossier_ref(an_id)
fetch_amendements_for_dossier(dossier_ref, chambre, num, settings)
if progress:
bar.update(step=len(dossier_ref.lectures))
def put(local, remote, force=False):
user = client.context.get('user')
if client.cd:
remote = Path(client.cd) / remote
if not hasattr(local, 'read'):
local = Path(local)
if local.is_dir():
with unsudo(): # Force reset to SSH user.
mkdir(remote)
if user:
chown(user, remote)
for path in local.rglob('*'):
relative_path = path.relative_to(local)
put(path, remote / relative_path)
return
if not force and exists(remote):
lstat = os.stat(str(local))
rstat = client.sftp.stat(str(remote))
if (lstat.st_size == rstat.st_size
and lstat.st_mtime <= rstat.st_mtime):
print(f'{local} => {remote}: SKIPPING (reason: up to date)')
return
elif isinstance(local, StringIO):
local = BytesIO(local.read().encode())
if hasattr(local, 'read'):
func = client.sftp.putfo
bar = ProgressBar(prefix=f'Sending to {remote}',
animation='{spinner}',
template='{prefix} {animation} {done:B}')
else:
bar = ProgressBar(prefix=f'{local} => {remote}')
func = client.sftp.put
if client.dry_run:
print(bar.prefix)
return
tmp = str(Path('/tmp') / md5(str(remote).encode()).hexdigest())
try:
func(local,
tmp,
callback=lambda done, total: bar.update(done=done, total=total),
confirm=True)
except OSError as err:
print(red(f'Error while processing {remote}'))
print(red(err))
sys.exit(1)
if hasattr(local, 'read'):
bar.finish()
if user:
with (unsudo()):
with (sudo()):
chown(user, tmp)
mv(tmp, remote)
if user:
chown(user, remote)
def get(remote, local):
if client.cd:
remote = Path(client.cd) / remote
if hasattr(local, 'read'):
func = client.sftp.getfo
bar = ProgressBar(prefix=f'Reading from {remote}',
animation='{spinner}',
template='{prefix} {animation} {done:B}')
else:
bar = ProgressBar(prefix=f'{remote} => {local}',
template='{prefix} {animation} {percent} '
'({done:B}/{total:B}) ETA: {eta}')
func = client.sftp.get
func(str(remote), local,
callback=lambda done, total: bar.update(done=done, total=total))
if hasattr(local, 'read'):
local.seek(0)
bar.finish()
# progressist - Minimalist and pythonic progress bar
# PyPI: https://pypi.org/project/progressist/
# Github: https://github.com/pyrates/progressist
# pip install progressist
# Usage
from progressist import ProgressBar
bar = ProgressBar(total=mytotalstuff)
for item in mystuff:
# do_stuff
bar.update()