def _thread_body(self):
posted_data_time = time.time()
posted_anything_time = time.time()
ready_chunks = []
finished = None
while finished is None:
items = self._read_queue()
for item in items:
if isinstance(item, self.Finish):
finished = item
else:
# item is Chunk
ready_chunks.append(item)
cur_time = time.time()
if ready_chunks and cur_time - posted_data_time > self.rate_limit_seconds():
posted_data_time = cur_time
posted_anything_time = cur_time
self._send(ready_chunks)
ready_chunks = []
if cur_time - posted_anything_time > self.HEARTBEAT_INTERVAL_SECONDS:
posted_anything_time = cur_time
util.request_with_retry(self._client.post,
self._endpoint, json={'complete': False, 'failed': False})
# post the final close message. (item is self.Finish instance now)
util.request_with_retry(self._client.post,
self._endpoint, json={'complete': True, 'exitcode': int(finished.exitcode)})
def _send(self, chunks):
# create files dict. dict of <filename: chunks> pairs where chunks is a list of
# [chunk_id, chunk_data] tuples (as lists since this will be json).
files = {}
# Groupby needs group keys to be consecutive, so sort first.
chunks.sort(key=lambda c: c.filename)
for filename, file_chunks in itertools.groupby(chunks, lambda c: c.filename):
file_chunks = list(file_chunks) # groupby returns iterator
if filename not in self._file_policies:
self._file_policies[filename] = DefaultFilePolicy()
files[filename] = self._file_policies[filename].process_chunks(
file_chunks)
util.request_with_retry(
self._client.post, self._endpoint, json={'files': files})
def _thread_body(self):
posted_data_time = time.time()
posted_anything_time = time.time()
ready_chunks = []
finished = None
while finished is None:
items = self._read_queue()
for item in items:
if isinstance(item, self.Finish):
finished = item
else:
# item is Chunk
ready_chunks.append(item)
cur_time = time.time()
if ready_chunks and (finished or cur_time - posted_data_time >
self.rate_limit_seconds()):
posted_data_time = cur_time
posted_anything_time = cur_time
self._send(ready_chunks)
ready_chunks = []
if cur_time - posted_anything_time > self.heartbeat_seconds:
posted_anything_time = cur_time
self._handle_response(
util.request_with_retry(
self._client.post,
self._endpoint,
json={
"complete": False,
"failed": False
},
))
# post the final close message. (item is self.Finish instance now)
util.request_with_retry(
self._client.post,
self._endpoint,
json={
"complete": True,
"exitcode": int(finished.exitcode)
},
)
def _send(self, chunks):
# create files dict. dict of <filename: chunks> pairs where chunks is a list of
# [chunk_id, chunk_data] tuples (as lists since this will be json).
files = {}
# Groupby needs group keys to be consecutive, so sort first.
chunks.sort(key=lambda c: c.filename)
for filename, file_chunks in itertools.groupby(chunks, lambda c: c.filename):
file_chunks = list(file_chunks) # groupby returns iterator
# Specific file policies are set by internal/sender.py
self.set_default_file_policy(filename, DefaultFilePolicy())
files[filename] = self._file_policies[filename].process_chunks(file_chunks)
if not files[filename]:
del files[filename]
self._handle_response(
util.request_with_retry(
self._client.post, self._endpoint, json={"files": files}
)
)
