def _thread_body(self): posted_data_time = time.time() posted_anything_time = time.time() ready_chunks = [] finished = None while finished is None: items = self._read_queue() for item in items: if isinstance(item, self.Finish): finished = item else: # item is Chunk ready_chunks.append(item) cur_time = time.time() if ready_chunks and cur_time - posted_data_time > self.rate_limit_seconds(): posted_data_time = cur_time posted_anything_time = cur_time self._send(ready_chunks) ready_chunks = [] if cur_time - posted_anything_time > self.HEARTBEAT_INTERVAL_SECONDS: posted_anything_time = cur_time util.request_with_retry(self._client.post, self._endpoint, json={'complete': False, 'failed': False}) # post the final close message. (item is self.Finish instance now) util.request_with_retry(self._client.post, self._endpoint, json={'complete': True, 'exitcode': int(finished.exitcode)})
def _send(self, chunks): # create files dict. dict of <filename: chunks> pairs where chunks is a list of # [chunk_id, chunk_data] tuples (as lists since this will be json). files = {} # Groupby needs group keys to be consecutive, so sort first. chunks.sort(key=lambda c: c.filename) for filename, file_chunks in itertools.groupby(chunks, lambda c: c.filename): file_chunks = list(file_chunks) # groupby returns iterator if filename not in self._file_policies: self._file_policies[filename] = DefaultFilePolicy() files[filename] = self._file_policies[filename].process_chunks( file_chunks) util.request_with_retry( self._client.post, self._endpoint, json={'files': files})
def _thread_body(self): posted_data_time = time.time() posted_anything_time = time.time() ready_chunks = [] finished = None while finished is None: items = self._read_queue() for item in items: if isinstance(item, self.Finish): finished = item else: # item is Chunk ready_chunks.append(item) cur_time = time.time() if ready_chunks and (finished or cur_time - posted_data_time > self.rate_limit_seconds()): posted_data_time = cur_time posted_anything_time = cur_time self._send(ready_chunks) ready_chunks = [] if cur_time - posted_anything_time > self.heartbeat_seconds: posted_anything_time = cur_time self._handle_response( util.request_with_retry( self._client.post, self._endpoint, json={ "complete": False, "failed": False }, )) # post the final close message. (item is self.Finish instance now) util.request_with_retry( self._client.post, self._endpoint, json={ "complete": True, "exitcode": int(finished.exitcode) }, )
def _send(self, chunks): # create files dict. dict of <filename: chunks> pairs where chunks is a list of # [chunk_id, chunk_data] tuples (as lists since this will be json). files = {} # Groupby needs group keys to be consecutive, so sort first. chunks.sort(key=lambda c: c.filename) for filename, file_chunks in itertools.groupby(chunks, lambda c: c.filename): file_chunks = list(file_chunks) # groupby returns iterator # Specific file policies are set by internal/sender.py self.set_default_file_policy(filename, DefaultFilePolicy()) files[filename] = self._file_policies[filename].process_chunks(file_chunks) if not files[filename]: del files[filename] self._handle_response( util.request_with_retry( self._client.post, self._endpoint, json={"files": files} ) )