def _read_file_section(self, uuid, path, offset, length, gzipped): """ Reads length bytes of the file at the given path in the bundle. The result is gzipped if gzipped is True. """ if self._is_available_locally(uuid): file_path = self._get_target_path(uuid, path) string = file_util.read_file_section(file_path, offset, length) if gzipped: string = file_util.gzip_string(string) return string else: worker = self.get_bundle_worker(uuid) response_socket_id = self._worker_model.allocate_socket( worker['user_id'], worker['worker_id']) try: read_args = { 'type': 'read_file_section', 'offset': offset, 'length': length, } self._send_read_message(worker, response_socket_id, uuid, path, read_args) string = self._get_read_response_string(response_socket_id) finally: self._worker_model.deallocate_socket(response_socket_id) if not gzipped: string = file_util.un_gzip_string(string) return string
def read_file_section(self, uuid, path, offset, length, gzipped): """ Reads length bytes of the file at the given path in the bundle. The result is gzipped if gzipped is True. """ if self._is_available_locally(uuid): file_path = self._get_target_path(uuid, path) string = file_util.read_file_section(file_path, offset, length) if gzipped: string = file_util.gzip_string(string) return string else: worker = self._worker_model.get_bundle_worker(uuid) response_socket_id = self._worker_model.allocate_socket( worker['user_id'], worker['worker_id'] ) try: read_args = {'type': 'read_file_section', 'offset': offset, 'length': length} self._send_read_message(worker, response_socket_id, uuid, path, read_args) string = self._get_read_response_string(response_socket_id) finally: self._worker_model.deallocate_socket(response_socket_id) if not gzipped: string = file_util.un_gzip_string(string) return string
def summarize_file(self, uuid, path, num_head_lines, num_tail_lines, max_line_length, truncation_text, gzipped): """ Summarizes the file at the given path in the bundle, returning a string containing the given numbers of lines from beginning and end of the file. If the file needs to be truncated, places truncation_text at the truncation point. This string is gzipped if gzipped is True. """ if self._is_available_locally(uuid): file_path = self._get_target_path(uuid, path) string = file_util.summarize_file(file_path, num_head_lines, num_tail_lines, max_line_length, truncation_text) if gzipped: string = file_util.gzip_string(string) return string else: worker = self._worker_model.get_bundle_worker(uuid) response_socket_id = self._worker_model.allocate_socket(worker['user_id'], worker['worker_id']) try: read_args = { 'type': 'summarize_file', 'num_head_lines': num_head_lines, 'num_tail_lines': num_tail_lines, 'max_line_length': max_line_length, 'truncation_text': truncation_text, } self._send_read_message(worker, response_socket_id, uuid, path, read_args) string = self._get_read_response_string(response_socket_id) finally: self._worker_model.deallocate_socket(response_socket_id) if not gzipped: string = file_util.un_gzip_string(string) return string
def summarize_file_thread(final_path): string = gzip_string( summarize_file( final_path, args['num_head_lines'], args['num_tail_lines'], args['max_line_length'], args['truncation_text'], )) reply_fn(None, {}, string)
def summarize_file( self, uuid, path, num_head_lines, num_tail_lines, max_line_length, truncation_text, gzipped ): """ Summarizes the file at the given path in the bundle, returning a string containing the given numbers of lines from beginning and end of the file. If the file needs to be truncated, places truncation_text at the truncation point. This string is gzipped if gzipped is True. """ if self._is_available_locally(uuid): file_path = self._get_target_path(uuid, path) string = file_util.summarize_file( file_path, num_head_lines, num_tail_lines, max_line_length, truncation_text ) if gzipped: string = file_util.gzip_string(string) return string else: worker = self._worker_model.get_bundle_worker(uuid) response_socket_id = self._worker_model.allocate_socket( worker['user_id'], worker['worker_id'] ) try: read_args = { 'type': 'summarize_file', 'num_head_lines': num_head_lines, 'num_tail_lines': num_tail_lines, 'max_line_length': max_line_length, 'truncation_text': truncation_text, } self._send_read_message(worker, response_socket_id, uuid, path, read_args) string = self._get_read_response_string(response_socket_id) finally: self._worker_model.deallocate_socket(response_socket_id) if not gzipped: string = file_util.un_gzip_string(string) return string
def test_gzip_string(self): self.assertEqual(un_gzip_string(gzip_string('contents')), 'contents')
def read_file_section_thread(final_path): string = gzip_string( read_file_section(final_path, args['offset'], args['length'])) reply_fn(None, {}, string)
def test_single_local_gzip_path_remove_sources(self): source = os.path.join(self.temp_dir, 'filename.gz') self.write_bytes_to_file(gzip_string('testing'), source) self.do_upload([source], remove_sources=True) self.assertFalse(os.path.exists(source))
def test_single_local_gzip_path(self): source = os.path.join(self.temp_dir, 'filename.gz') self.write_bytes_to_file(gzip_string('testing'), source) self.do_upload([source], unpack=True) self.assertTrue(os.path.exists(source)) self.check_file_contains_string(self.bundle_location, 'testing')