def _add_pending(self, relpath_u):
self._log("add pending %r" % (relpath_u,))
if magicpath.should_ignore_file(relpath_u):
self._log("_add_pending %r but should_ignore()==True" % (relpath_u,))
return
if relpath_u in self._pending:
self._log("_add_pending %r but already pending" % (relpath_u,))
return
self._pending.add(relpath_u)
progress = PercentProgress()
item = UploadItem(relpath_u, progress)
item.set_status('queued', self._clock.seconds())
self._deque.append(item)
self._count('objects_queued')
self._log("_add_pending(%r) queued item" % (relpath_u,))
def _filter_batch_to_deque(ign):
self._log("deque = %r, scan_batch = %r" % (self._deque, scan_batch))
for relpath_u in scan_batch.keys():
file_node, metadata = max(scan_batch[relpath_u], key=lambda x: x[1]['version'])
if self._should_download(relpath_u, metadata['version'], file_node.get_readonly_uri()):
to_dl = DownloadItem(
relpath_u,
PercentProgress(file_node.get_size()),
file_node,
metadata,
)
to_dl.set_status('queued', self._clock.seconds())
self._deque.append(to_dl)
self._count("objects_queued")
else:
self._log("Excluding %r" % (relpath_u,))
self._call_hook(None, 'processed', async=True) # await this maybe-Deferred??
self._log("deque after = %r" % (self._deque,))
def a_progress(progress, total_size):
p = PercentProgress(total_size)
p.set_progress(progress)
return p